From 63c81b6d05684fa5198d7cba5ade575d7caacd06 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 30 Aug 2024 14:03:01 +0100 Subject: [PATCH 01/23] Split off 9.8.18 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5184e82c412..55ee3326270 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.8.17-closedown0 +version: 9.8.19-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.8.17-closedown0 +appVersion: 9.8.19-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 0ab1e9a8c97..2ff2aa6ba3e 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.8.17-closedown0 +helmVersion: 9.8.19-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 9fe8c6016cb..5dceb8dab97 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 8 ) -set ( HPCC_POINT 17 ) +set ( HPCC_POINT 19 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-23T14:16:28Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-30T13:03:01Z" ) ### From 6d2ccc90ea61ecac9d198ae187d956abbf72dacd Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 30 Aug 2024 14:03:58 +0100 Subject: [PATCH 02/23] Split off 9.6.44 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 621dd5aeaa3..d40fb35fc9c 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.43-closedown0 +version: 9.6.45-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.43-closedown0 +appVersion: 9.6.45-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index f2dc4a62378..6ad1f8dd260 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.6.43-closedown0 +helmVersion: 9.6.45-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index fc715fee250..6fd7c9715fc 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 43 ) +set ( HPCC_POINT 45 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-23T14:17:52Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-30T13:03:58Z" ) ### From 395a4b03739d076e6c6befbb0e1ca7e47df45461 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 30 Aug 2024 14:04:55 +0100 Subject: [PATCH 03/23] Split off 9.4.92 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index ed35b391f22..4bb350cda87 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.91-closedown0 +version: 9.4.93-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.91-closedown0 +appVersion: 9.4.93-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index a499348431a..80b13942c50 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.91-closedown0 +helmVersion: 9.4.93-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 922ad4495ee..a846e65ffab 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 91 ) +set ( HPCC_POINT 93 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-23T14:19:09Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-30T13:04:55Z" ) ### From e8c2224216b6a397db256dfc54b9c684045905ad Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 30 Aug 2024 14:05:45 +0100 Subject: [PATCH 04/23] Split off 9.2.118 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 6e9c6196cc2..32bff60c6ba 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.117-closedown0 +version: 9.2.119-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.117-closedown0 +appVersion: 9.2.119-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index f5348423f25..4528eafa841 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.117-closedown0 +helmVersion: 9.2.119-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 3eabbd4c89c..8e35b51b00c 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 117 ) +set ( HPCC_POINT 119 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-08-23T14:20:10Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-08-30T13:05:45Z" ) ### From 3e438156966d1e386999a04934224ac763ef5931 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Wed, 28 Aug 2024 14:45:57 -0400 Subject: [PATCH 05/23] HPCC-32411 Update GrafanaLogAccess plugin name - Updates GrafanaLogAccess Plugin name Signed-off-by: Rodrigo Pastrana --- system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp index 5953161f5a9..316dd097ad8 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp @@ -55,7 +55,7 @@ static constexpr int defaultEntryStart = 0; class GrafanaLogAccessCurlClient : public CInterfaceOf { private: - static constexpr const char * type = "grafanaloganalyticscurl"; + static constexpr const char * type = "grafanacurl"; Owned m_pluginCfg; StringBuffer m_grafanaConnectionStr; GrafanaDataSource m_targetDataSource; From 4da6e826b39d95b0645f4dc8591333b3dd36e2c3 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 2 Sep 2024 12:02:46 +0100 Subject: [PATCH 06/23] HPCC-32553 Ensure errors are reported when closing files Signed-off-by: Gavin Halliday --- dali/base/dacoven.cpp | 2 + dali/base/dasds.cpp | 2 + dali/server/daserver.cpp | 1 + ecl/ecl-bundle/ecl-bundle.cpp | 1 + ecl/eclcc/eclcc.cpp | 10 ++++- ecl/eclcmd/queries/ecl-queries.cpp | 3 ++ ecl/hql/hqlcache.cpp | 1 + ecl/hqlcpp/hqlecl.cpp | 4 +- ecl/hqlcpp/hqlwcpp.cpp | 17 +++++--- ecl/hqlcpp/hqlwcpp.hpp | 4 +- ecl/hqlcpp/hqlwcpp.ipp | 14 +------ ecl/hthor/hthor.cpp | 1 + system/jlib/jcomp.cpp | 4 ++ system/jlib/jfile.cpp | 17 +++++++- system/jlib/jlzw.cpp | 64 ++++++++++++++++++------------ system/jlib/jptree.cpp | 2 + system/jlib/jutil.cpp | 2 + thorlcr/master/thgraphmanager.cpp | 1 + thorlcr/slave/slavmain.cpp | 1 + thorlcr/thorutil/thormisc.cpp | 1 + 20 files changed, 101 insertions(+), 51 deletions(-) diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp index 081f5f2d73c..8aa60877944 100644 --- a/dali/base/dacoven.cpp +++ b/dali/base/dacoven.cpp @@ -563,12 +563,14 @@ class CCovenServer: public CCovenBase Owned f = createIFile(storename.get()); Owned io = f->open(IFOcreate); io->write(0, xml.length(), xml.str()); + io->close(); io.clear(); if (!backupname.isEmpty()) { try { f.setown(createIFile(backupname.get())); io.setown(f->open(IFOcreate)); io->write(0, xml.length(), xml.str()); + io->close(); io.clear(); } catch (IException *e) { diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index 89caaa9efc5..2f6410172be 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -1022,6 +1022,7 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re sprintf(strNum, "%016" I64F "X", fLen); memcpy(headerPtr + deltaHeaderSizeOff, strNum, 16); iFileIO->write(0, strlen(deltaHeader), headerPtr); + iFileIO->close(); } catch (IException *e) { @@ -1213,6 +1214,7 @@ class CDeltaWriter : implements IThreaded Owned iFile = createIFile(rL.str()); Owned fileIO = iFile->open(IFOcreate); fileIO->write(0, length, data); + fileIO->close(); } catch (IException *e) { diff --git a/dali/server/daserver.cpp b/dali/server/daserver.cpp index cdd2f21a1e9..f0004457432 100644 --- a/dali/server/daserver.cpp +++ b/dali/server/daserver.cpp @@ -612,6 +612,7 @@ int main(int argc, const char* argv[]) backupCheck.append("bakchk.").append((unsigned)GetCurrentProcessId()); OwnedIFile iFileDataDir = createIFile(backupCheck.str()); OwnedIFileIO iFileIO = iFileDataDir->open(IFOcreate); + iFileIO->close(); iFileIO.clear(); try { diff --git a/ecl/ecl-bundle/ecl-bundle.cpp b/ecl/ecl-bundle/ecl-bundle.cpp index 7a9074905e4..bebbc1e4d24 100644 --- a/ecl/ecl-bundle/ecl-bundle.cpp +++ b/ecl/ecl-bundle/ecl-bundle.cpp @@ -750,6 +750,7 @@ class CBundleInfoSet : public CInterfaceOf VStringBuffer redirect("IMPORT %s.%s.%s.%s as _%s; EXPORT %s := _%s;", VERSION_SUBDIR, name, version, name, name, name, name); Owned rfile = redirector->open(IFOcreate); rfile->write(0, redirect.length(), redirect.str()); + rfile->close(); bundle->setActive(true); } virtual void setActive(const char *version) diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp index 14e7d101633..e898cabece3 100644 --- a/ecl/eclcc/eclcc.cpp +++ b/ecl/eclcc/eclcc.cpp @@ -1990,8 +1990,11 @@ void EclCC::outputXmlToOutputFile(EclCompileInstance & instance, IPropertyTree * { //Work around windows problem writing 64K to stdout if not redirected/piped Owned stream = createIOStream(ifileio.get()); - Owned buffered = createBufferedIOStream(stream,0x8000); - saveXML(*buffered, xml); + { + Owned buffered = createBufferedIOStream(stream,0x8000); + saveXML(*buffered, xml); + } + ifileio->close(); } } @@ -2044,7 +2047,10 @@ void EclCC::generateOutput(EclCompileInstance & instance) OwnedIFileIO ifileio = createArchiveOutputFile(instance); if (ifileio) + { ifileio->write(0, filenames.length(), filenames.str()); + ifileio->close(); + } } else { diff --git a/ecl/eclcmd/queries/ecl-queries.cpp b/ecl/eclcmd/queries/ecl-queries.cpp index 34e0cfa7030..5a59f94a20b 100644 --- a/ecl/eclcmd/queries/ecl-queries.cpp +++ b/ecl/eclcmd/queries/ecl-queries.cpp @@ -1382,7 +1382,10 @@ class EclCmdQueriesExport : public EclCmdCommon fprintf(stdout, "\nWriting to file %s\n", file->queryFilename()); if (io.get()) + { io->write(0, strlen(s), s); + io->close(); + } else fprintf(stderr, "\nFailed to create file %s\n", file->queryFilename()); } diff --git a/ecl/hql/hqlcache.cpp b/ecl/hql/hqlcache.cpp index 1e9cd46ebd0..56a6b24f32f 100644 --- a/ecl/hql/hqlcache.cpp +++ b/ecl/hql/hqlcache.cpp @@ -492,6 +492,7 @@ static void extractFile(const char * path, const char * moduleName, const char * Owned io = file->open(IFOcreate); if (text) io->write(0, strlen(text), text); + io->close(); io.clear(); if (ts) { diff --git a/ecl/hqlcpp/hqlecl.cpp b/ecl/hqlcpp/hqlecl.cpp index e52ac773ca9..ddd6c3e63c3 100644 --- a/ecl/hqlcpp/hqlecl.cpp +++ b/ecl/hqlcpp/hqlecl.cpp @@ -310,7 +310,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl addDirectoryPrefix(fullname, targetDir).append(filename); Owned out = createIFile(fullname.str()); - Owned expander = createTemplateExpander(out, codeTemplate); + Owned expander = createTemplateExpander(codeTemplate); Owned writer = createCppWriter(*code, compiler); Owned props = createProperties(true); @@ -324,7 +324,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl props->setProp("headerName", headerName.str()); props->setProp("outputName", fullname.str()); - expander->generate(*writer, pass, props); + expander->generate(*writer, out, pass, props); totalGeneratedSize += out->size(); diff --git a/ecl/hqlcpp/hqlwcpp.cpp b/ecl/hqlcpp/hqlwcpp.cpp index 0adcc71d34c..ef59a67735c 100644 --- a/ecl/hqlcpp/hqlwcpp.cpp +++ b/ecl/hqlcpp/hqlwcpp.cpp @@ -103,9 +103,14 @@ CppWriterTemplate::CppWriterTemplate(const char * codeTemplate) loadTemplate(codeTemplate); } -void CppWriterTemplate::generate(ISectionWriter & writer, unsigned pass, IProperties * properties) +void CppWriterTemplate::generate(ISectionWriter & writer, IFile * outputFile, unsigned pass, IProperties * properties) { - writer.setOutput(out, outStream); + Owned io = outputFile->open(IFOcreate); + if (!io) + throwError1(HQLERR_CouldNotCreateOutputX, outputFile->queryFilename()); + + outStream.setown(createIOStream(io)); + writer.setOutput(outputFile, outStream); const char * finger = text; bool output = true; @@ -149,6 +154,8 @@ void CppWriterTemplate::generate(ISectionWriter & writer, unsigned pass, IProper outputQuoted(writer, end-finger, finger); writer.setOutput(NULL, NULL); + outStream.clear(); + io->close(); } void CppWriterTemplate::loadTemplate(const char * codeTemplate) @@ -2271,11 +2278,9 @@ void HqlCppSectionWriter::generateSection(unsigned delta, IAtom * section, unsig //--------------------------------------------------------------------------- -ITemplateExpander * createTemplateExpander(IFile * output, const char * codeTemplate) +ITemplateExpander * createTemplateExpander(const char * codeTemplate) { - Owned expander = new CppWriterTemplate(codeTemplate); - expander->setOutput(output); - return expander.getClear(); + return new CppWriterTemplate(codeTemplate); } ISectionWriter * createCppWriter(IHqlCppInstance & _instance, CompilerType compiler) diff --git a/ecl/hqlcpp/hqlwcpp.hpp b/ecl/hqlcpp/hqlwcpp.hpp index 535c41374b8..e8001cc1de3 100644 --- a/ecl/hqlcpp/hqlwcpp.hpp +++ b/ecl/hqlcpp/hqlwcpp.hpp @@ -35,7 +35,7 @@ interface HQLCPP_API ISectionWriter : public IInterface interface HQLCPP_API ITemplateExpander : public IInterface { public: - virtual void generate(ISectionWriter & writer, unsigned pass, IProperties * properties = NULL) = 0; + virtual void generate(ISectionWriter & writer, IFile * _output, unsigned pass, IProperties * properties = NULL) = 0; }; extern HQLCPP_API StringBuffer & generateExprCpp(StringBuffer & out, IHqlExpression * expr, CompilerType compiler); @@ -43,7 +43,7 @@ extern HQLCPP_API StringBuffer & generateTypeCpp(StringBuffer & out, ITypeInfo * bool generateFunctionPrototype(StringBuffer & out, IHqlExpression * funcdef, CompilerType compiler); void generateFunctionReturnType(StringBuffer & prefix, StringBuffer & params, ITypeInfo * retType, IHqlExpression * attrs, CompilerType compiler); -extern HQLCPP_API ITemplateExpander * createTemplateExpander(IFile * output, const char * codeTemplate); +extern HQLCPP_API ITemplateExpander * createTemplateExpander(const char * codeTemplate); extern HQLCPP_API ISectionWriter * createCppWriter(IHqlCppInstance & _instance, CompilerType compiler); extern bool isTypePassedByAddress(ITypeInfo * type); diff --git a/ecl/hqlcpp/hqlwcpp.ipp b/ecl/hqlcpp/hqlwcpp.ipp index 8074016b25f..613c5da19fd 100644 --- a/ecl/hqlcpp/hqlwcpp.ipp +++ b/ecl/hqlcpp/hqlwcpp.ipp @@ -23,7 +23,7 @@ public: CppWriterTemplate(const char * codeTemplate); IMPLEMENT_IINTERFACE - virtual void generate(ISectionWriter & writer, unsigned pass, IProperties * properties = NULL); + virtual void generate(ISectionWriter & writer, IFile * _output, unsigned pass, IProperties * properties = NULL); void outputQuoted(ISectionWriter & writer, size32_t len, const char * str) { @@ -31,17 +31,6 @@ public: outStream->write(len, str); } - void setOutput(IFile * _output) - { - Owned io = _output->open(IFOcreate); - if (!io) - throwError1(HQLERR_CouldNotCreateOutputX, _output->queryFilename()); - - out.set(_output); - outStream.setown(createIOStream(io)); - } - - private: void loadTemplate(const char * codeTemplate); enum TplSectionType { TplEmbed, TplExpand, TplCondition, TplEndCondition }; @@ -58,7 +47,6 @@ protected: const char * text; unsigned len; CIArray sections; - Owned out; Owned outStream; }; diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 166cea2c0a7..c4820125790 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -1295,6 +1295,7 @@ void CHThorIndexWriteActivity::execute() offsetBranches = builder->getOffsetBranches(); out->flush(); out.clear(); + io->close(); } if(clusterHandler) diff --git a/system/jlib/jcomp.cpp b/system/jlib/jcomp.cpp index 0e86580ccc8..9f6175c7edf 100644 --- a/system/jlib/jcomp.cpp +++ b/system/jlib/jcomp.cpp @@ -475,7 +475,10 @@ void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log) Owned fio = f->open(IFOcreate); if(fio.get()) + { fio->write(0, log.length(), log.str()); + fio->close(); + } } bool CppCompiler::compile() @@ -560,6 +563,7 @@ bool CppCompiler::compile() //Don't leave lots of blank log files around if the compile was successful bool logIsEmpty = (dstIO->size() == 0); + dstIO->close(); dstIO.clear(); if (ret && logIsEmpty) dstfile->remove(); diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index b7308a06850..95cc17ce45a 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -73,6 +73,9 @@ // this should not be enabled in WindowRemoteDirectory used //#define CHECK_FILE_IO // If enabled, reads and writes are checked for sensible parameters +#ifdef _DEBUG +//#define CHECK_FILE_CLOSED_BEFORE_DELETE +#endif #ifdef _DEBUG #define ASSERTEX(e) assertex(e); @@ -2075,13 +2078,22 @@ CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags CFileIO::~CFileIO() { +#ifdef CHECK_FILE_CLOSED_BEFORE_DELETE + //Any file that is being written to, should be closed before the object is destroyed, otherwise errors from failing to commit will be lost + if ((file != NULLFILE) && (openmode!=IFOread)) + { + OERRLOG("CFileIO::~CFileIO - file object destroyed without being closed first"); // A programmer problem, but the operator should know about it. + PrintStackReport(); + } +#endif try { close(); } catch (IException * e) { - EXCLOG(e, "CFileIO::~CFileIO"); + //An error closing a file cannot throw an exception, but should be logged as a very severe error in the logs. + DISLOG(e, "CFileIO::~CFileIO"); PrintStackReport(); e->Release(); } @@ -3289,6 +3301,7 @@ void doCopyFile(IFile * target, IFile * source, size32_t buffersize, ICopyFilePr if (progress && progress->onProgress(offset, total) != CFPcontinue) break; } + targetIO->close(); // Ensure errors are reported. targetIO.clear(); if (usetmp) { StringAttr tail(pathTail(target->queryFilename())); @@ -4441,6 +4454,7 @@ void touchFile(IFile *iFile) Owned iFileIO = iFile->open(IFOcreate); if (!iFileIO) throw makeStringExceptionV(0, "touchFile: failed to create file %s", iFile->queryFilename()); + iFileIO->close(); } void touchFile(const char *filename) @@ -7323,6 +7337,7 @@ extern jlib_decl void writeSentinelFile(IFile * sentinelFile) { Owned sentinel = sentinelFile->open(IFOcreate); sentinel->write(0, 5, "rerun"); + sentinel->close(); } catch(IException *E) { diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index d0fb75762ff..5629865855e 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2077,39 +2077,51 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface //If the blocks are being expanded incrementally check if the position is within the current block //This test will never be true for row compressed data, or non-incremental decompression - if ((pos >= startBlockPos) && (pos < startBlockPos + fullBlockSize)) + try { - if (pos < nextExpansionPos) + if ((pos >= startBlockPos) && (pos < startBlockPos + fullBlockSize)) { - //Start decompressing again and avoid re-reading the data from disk - const void * rawData; - if (fileio) - rawData = compressedInputBlock.get(); - else - rawData = mmfile->base()+startBlockPos; - - assertex(rawData); - size32_t exp = expander->expandFirst(curblockbuf, rawData); - curblockpos = startBlockPos; - nextExpansionPos = startBlockPos + exp; if (pos < nextExpansionPos) - return; + { + //Start decompressing again and avoid re-reading the data from disk + const void * rawData; + if (fileio) + rawData = compressedInputBlock.get(); + else + rawData = mmfile->base()+startBlockPos; + + assertex(rawData); + nextExpansionPos = startBlockPos; // update in case an exception is thrown + size32_t exp = expander->expandFirst(curblockbuf, rawData); + curblockpos = startBlockPos; + nextExpansionPos = startBlockPos + exp; + if (pos < nextExpansionPos) + return; - curblockbuf.clear(); - } + curblockbuf.clear(); + } - for (;;) - { - size32_t nextSize = expander->expandNext(curblockbuf); - if (nextSize == 0) - throw makeStringExceptionV(-1, "Unexpected zero length compression block at position %llu", curblockpos); + for (;;) + { + size32_t nextSize = expander->expandNext(curblockbuf); + if (nextSize == 0) + throw makeStringException(-1, "Unexpected zero length compression block"); - curblockpos = nextExpansionPos; - nextExpansionPos = nextExpansionPos+nextSize; - if (pos < nextExpansionPos) - return; + curblockpos = nextExpansionPos; + nextExpansionPos = nextExpansionPos+nextSize; + if (pos < nextExpansionPos) + return; + } } } + catch (IException * e) + { + unsigned code = e->errorCode(); + StringBuffer msg; + e->errorMessage(msg).appendf(" at position %llu of %llu", nextExpansionPos, trailer.indexPos); + e->Release(); + throw makeStringException(code, msg.str()); + } size32_t expsize; curblocknum = lookupIndex(pos,curblockpos,expsize); @@ -2456,6 +2468,8 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface } checkedwrite(trailer.indexPos,indexbuf.length(),indexbuf.toByteArray()); indexbuf.clear(); + if (fileio) + fileio->close(); } mode = ICFread; curblockpos = 0; diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index d7ba2871c78..70d483495ad 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -6332,6 +6332,7 @@ void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned if (!ifileio) throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename()); saveXML(*ifileio, tree, indent, flags); + ifileio->close(); // Ensure errors are reported } void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags) @@ -9692,6 +9693,7 @@ void saveYAML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned if (!ifileio) throw MakeStringException(0, "saveXML: could not find %s to open", ifile.queryFilename()); saveYAML(*ifileio, tree, indent, flags); + ifileio->close(); } void saveYAML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags) diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp index d5c8acb8917..109945c56c6 100644 --- a/system/jlib/jutil.cpp +++ b/system/jlib/jutil.cpp @@ -3422,6 +3422,7 @@ void jlib_decl atomicWriteFile(const char *fileName, const char *output) if (!ifileio) throw MakeStringException(0, "atomicWriteFile: could not create output file %s", newFileName.str()); ifileio->write(0, strlen(output), output); + ifileio->close(); } #else VStringBuffer newFileName("%s.XXXXXX", fileName); @@ -3435,6 +3436,7 @@ void jlib_decl atomicWriteFile(const char *fileName, const char *output) if (!ifileio) throw MakeStringException(0, "atomicWriteFile: could not create output file %s", newFileName.str()); ifileio->write(0, strlen(output), output); + ifileio->close(); } #endif if (file->exists()) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index ce4f63bd4fa..dc78ae4c416 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1083,6 +1083,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, out->setCreateFlags(S_IRWXU); OwnedIFileIO io = out->open(IFOcreate); io->write(0, file.length(), file.toByteArray()); + io->close(); io.clear(); } catch (IException *e) diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index d3eb593354b..325312fe10d 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1831,6 +1831,7 @@ class CJobListener : public CSimpleInterface iFile->setCreateFlags(S_IRWXU); Owned iFileIO = iFile->open(IFOwrite); iFileIO->write(0, size, queryPtr); + iFileIO->close(); } catch (IException *e) { diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 2aaf9e65914..afc06cee34a 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -1700,4 +1700,5 @@ void saveWuidToFile(const char *wuid) if (!wuidFileIO) throw makeStringException(0, "Failed to create file 'wuid' to store current workunit for post mortem script"); wuidFileIO->write(0, strlen(wuid), wuid); + wuidFileIO->close(); } From aa2dcbae48c877250c887268185541fad6c31ad0 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 28 Aug 2024 12:14:15 +0100 Subject: [PATCH 07/23] HPCC-32564 Limit rhs of JOIN to 1 match converting LOOKUP JOIN to ALL JOIN Signed-off-by: Gavin Halliday --- ecl/hql/hqlopt.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ecl/hql/hqlopt.cpp b/ecl/hql/hqlopt.cpp index 001acae84ae..bd4f5adcf98 100644 --- a/ecl/hql/hqlopt.cpp +++ b/ecl/hql/hqlopt.cpp @@ -4285,6 +4285,22 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme args.replace(*createConstant(true), 2); } + //This test currently duplicates the test in the enclosing if. This is in preparation for applying the logic + //to other joins later by removing the outer test. + if (transformed->hasAttribute(lookupAtom)) + { + //LOOKUP join implicitly dedups the right hand side by the hard-match-condition (unless MANY is specified). + //Technically this is not equivalent to KEEP(1) - since that is only applied after any soft-match condition + //However in this case the match condition is true (or row-invarient in very obscure cases), + //so they are identical and this transformation is valid. + if (!transformed->hasAttribute(manyAtom)) + { + //There should never be a KEEP attribute if MANY not specified - but remove in case. + removeAttribute(args, keepAtom); + args.append(*createAttribute(keepAtom, createConstant(1))); + } + removeAttribute(args, lookupAtom); + } args.append(*createAttribute(allAtom)); } if (doTrace(traceOptimizations)) From 3ec725e79987a9eec5788f0c849f279dcd70e36b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 2 Sep 2024 16:45:09 +0100 Subject: [PATCH 08/23] HPCC-32584 Ensure that dfuserver closes files cleanly so errors are reported Signed-off-by: Gavin Halliday --- common/dllserver/thorplugin.cpp | 2 ++ dali/base/dasds.cpp | 2 ++ dali/ft/filecopy.cpp | 1 + dali/ft/ftbase.ipp | 4 ++++ dali/ft/fttransform.cpp | 9 +++++++++ system/jlib/jfcmp.hpp | 6 ++++++ system/jlib/jfile.cpp | 17 ++++++++++++++++- system/jlib/jfile.hpp | 1 + system/jlib/jfile.ipp | 2 ++ tools/dumpkey/dumpkey.cpp | 1 + 10 files changed, 44 insertions(+), 1 deletion(-) diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index 81299faf66d..939cfc071ee 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -559,6 +559,8 @@ const StringArray &HelperDll::queryManifestFiles(const char *type, const char *w OwnedIFileIO o = f->open(IFOcreate); assertex(o.get() != nullptr); o->write(0, len, data); + o->close(); + list->append(extractName); if (doTrace(traceJava) && streq(type, "jar")) DBGLOG("Extracted jar resource %u size %u to %s in %u ms", id, len, extractName.str(), msTick() - start); diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index 2f6410172be..5be65389400 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -5289,6 +5289,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface iFileIO->write(0, sizeof(unsigned), crc); if (storeInfo) storeInfo->cache.set(filename.str()); + iFileIO->close(); } void updateStoreInfo(const char *base, const char *location, unsigned edition, unsigned *crc, CStoreInfo *storeInfo=NULL) @@ -5400,6 +5401,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface OwnedIFileIO detachIPIO = detachIPIFile->open(IFOcreate); detachIPIO->write(0, sizeof(storeHelper.mySessId), &storeHelper.mySessId); + detachIPIO->close(); detachIPIO.clear(); detachIPIFile->rename(activeDetachIPStr.str()); // check often do not wait any longer than necessary diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 6834135bffa..ad88915c46c 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -1023,6 +1023,7 @@ void FileSprayer::beforeTransfer() io->write(lastOutputOffset-sizeof(null), sizeof(null), &null); } } + io->close(); } } } diff --git a/dali/ft/ftbase.ipp b/dali/ft/ftbase.ipp index 2c6ea512d30..70c472a612a 100644 --- a/dali/ft/ftbase.ipp +++ b/dali/ft/ftbase.ipp @@ -134,6 +134,10 @@ public: virtual offset_t tell() override; virtual size32_t write(size32_t len, const void * data) override; virtual unsigned __int64 getStatistic(StatisticKind kind) override { return stream->getStatistic(kind); } + virtual void close() override + { + stream->close(); + } unsigned getCRC() { return crc; } void setCRC(unsigned long _crc) { crc = _crc; } diff --git a/dali/ft/fttransform.cpp b/dali/ft/fttransform.cpp index 4e8ca2fdec8..db88821696e 100644 --- a/dali/ft/fttransform.cpp +++ b/dali/ft/fttransform.cpp @@ -790,6 +790,8 @@ bool TransferServer::pull() assertex(curProgress.status != OutputProgress::StatusRenamed); if (curProgress.status != OutputProgress::StatusCopied) { + if (out) + out->close(); out.setown(createIOStream(outio)); out->seek(progressOffset, IFSbegin); wrapOutInCRC(curProgress.outputCRC); @@ -885,6 +887,8 @@ bool TransferServer::pull() } } + if (out) + out->close(); out.setown(createIOStream(outio)); out->seek(0, IFSbegin); wrapOutInCRC(0); @@ -903,7 +907,10 @@ bool TransferServer::pull() } crcOut.clear(); + if (out) + out->close(); out.clear(); + //Once the transfers have completed, rename the files, and sync file times //if replicating... if (!isSafeMode) @@ -997,6 +1004,7 @@ bool TransferServer::push() } outio.setown(createCompressedFileWriter(outio, false, 0, true, compressor, COMPRESS_METHOD_LZ4)); } + out.setown(createIOStream(outio)); if (!compressOutput) out->seek(curPartition.outputOffset + curProgress.outputLength, IFSbegin); @@ -1011,6 +1019,7 @@ bool TransferServer::push() sendProgress(curProgress); } crcOut.clear(); + out->close(); out.clear(); } } diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index 1e355bd4855..e5a84377555 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -440,6 +440,12 @@ class CFcmpStream : public CSimpleInterfaceOf { return baseio->getStatistic(kind); } + + virtual void close() override + { + flush(); + baseio->close(); + } }; #endif diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 95cc17ce45a..115a67c665e 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -2953,7 +2953,11 @@ class CBufferedFileIOStream : public CBufferedFileIOStreamBase { return io->getStatistic(kind); } - + virtual void close() override + { + flush(); + io->close(); + } protected: IFileIOAttr io; }; @@ -3064,6 +3068,13 @@ class CBufferedAsyncIOStream: public CBufferedFileIOStreamBase virtual size32_t directWrite(size32_t len, const void * data) { assertex(false); return 0; } // shouldn't get called virtual offset_t directSize() { waitAsyncWrite(); return io->size(); } virtual unsigned __int64 getStatistic(StatisticKind kind) { return io->getStatistic(kind); } + virtual void close() override + { + flush(); + waitAsyncWrite(); + waitAsyncRead(); + io->close(); + } }; @@ -4571,6 +4582,10 @@ IFileIOStream *createProgressIFileIOStream(IFileIOStream *iFileIOStream, offset_ { return iFileIOStream->getStatistic(kind); } + virtual void close() override + { + iFileIOStream->close(); + } }; return new CProgressIFileIOStream(iFileIOStream, totalSize, msg, periodSecs); } diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index 0020f759ef7..c55c5b7ccdd 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -222,6 +222,7 @@ interface IFileIOStream : extends IIOStream virtual offset_t size() = 0; virtual offset_t tell() = 0; virtual unsigned __int64 getStatistic(StatisticKind kind) = 0; + virtual void close() = 0; }; interface IDiscretionaryLock: extends IInterface diff --git a/system/jlib/jfile.ipp b/system/jlib/jfile.ipp index dfeef77ae48..56682a53bcf 100644 --- a/system/jlib/jfile.ipp +++ b/system/jlib/jfile.ipp @@ -219,6 +219,7 @@ public: virtual offset_t tell(); virtual size32_t write(size32_t len, const void * data); virtual unsigned __int64 getStatistic(StatisticKind kind) { return io->getStatistic(kind); } + virtual void close() override { io->close(); } protected: Linked io; offset_t curOffset; @@ -238,6 +239,7 @@ public: virtual offset_t tell(); virtual size32_t write(size32_t len, const void * data); virtual unsigned __int64 getStatistic(StatisticKind kind) { return stream->getStatistic(kind); } + virtual void close() override { stream->close(); } protected: Linked stream; }; diff --git a/tools/dumpkey/dumpkey.cpp b/tools/dumpkey/dumpkey.cpp index 88ef41f7a31..1b3c862fd2f 100644 --- a/tools/dumpkey/dumpkey.cpp +++ b/tools/dumpkey/dumpkey.cpp @@ -183,6 +183,7 @@ class DummyFileIOStream : public CInterfaceOf virtual offset_t size() override { return hwm; } virtual offset_t tell() override { return offset; } virtual unsigned __int64 getStatistic(StatisticKind kind) { return stats.getStatistic(kind); } + virtual void close() override { } private: offset_t offset = 0; offset_t hwm = 0; From 8241885d5dc4df12802b7d79c739f697ca1e0d8a Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Tue, 3 Sep 2024 13:00:02 +0000 Subject: [PATCH 09/23] HPCC-32448 Parquet Plugin defaults to incorrect size for some types --- plugins/parquet/parquetembed.cpp | 75 ++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index 2808e699dc8..18bd4c00e07 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -743,10 +743,18 @@ arrow::Status ParquetWriter::fieldToNode(const RtlFieldInfo *field, std::vector< { arrowFields.push_back(std::make_shared(name.str(), arrow::int64())); } - else + else if (field->type->length > 2) { arrowFields.push_back(std::make_shared(name.str(), arrow::int32())); } + else if (field->type->length > 1) + { + arrowFields.push_back(std::make_shared(name.str(), arrow::int16())); + } + else + { + arrowFields.push_back(std::make_shared(name.str(), arrow::int8())); + } } else { @@ -754,14 +762,29 @@ arrow::Status ParquetWriter::fieldToNode(const RtlFieldInfo *field, std::vector< { arrowFields.push_back(std::make_shared(name.str(), arrow::uint64())); } - else + else if (field->type->length > 2) { arrowFields.push_back(std::make_shared(name.str(), arrow::uint32())); } + else if (field->type->length > 1) + { + arrowFields.push_back(std::make_shared(name.str(), arrow::uint16())); + } + else + { + arrowFields.push_back(std::make_shared(name.str(), arrow::uint8())); + } } break; case type_real: - arrowFields.push_back(std::make_shared(name.str(), arrow::float64())); + if (field->type->length > 4) + { + arrowFields.push_back(std::make_shared(name.str(), arrow::float64())); + } + else + { + arrowFields.push_back(std::make_shared(name.str(), arrow::float32())); + } break; case type_char: case type_string: @@ -770,8 +793,17 @@ arrow::Status ParquetWriter::fieldToNode(const RtlFieldInfo *field, std::vector< case type_utf8: case type_unicode: case type_varunicode: + if (field->type->length > 4 || field->type->length == 0) + { + arrowFields.push_back(std::make_shared(name.str(), arrow::large_utf8())); + } + else + { + arrowFields.push_back(std::make_shared(name.str(), arrow::utf8())); + } + break; case type_decimal: - arrowFields.push_back(std::make_shared(name.str(), arrow::utf8())); // TODO: add decimal encoding + arrowFields.push_back(std::make_shared(name.str(), arrow::large_utf8())); // TODO: add decimal encoding break; case type_data: if (field->type->length > 0) @@ -1006,6 +1038,12 @@ void ParquetWriter::addFieldToBuilder(const RtlFieldInfo *field, unsigned len, c reportIfFailure(stringBuilder->Append(data, len)); break; } + case arrow::Type::LARGE_STRING: + { + arrow::LargeStringBuilder *largeStringBuilder = static_cast(fieldBuilder); + reportIfFailure(largeStringBuilder->Append(data, len)); + break; + } case arrow::Type::LARGE_BINARY: { arrow::LargeBinaryBuilder *largeBinaryBuilder = static_cast(fieldBuilder); @@ -1763,6 +1801,18 @@ void ParquetRecordBinder::processInt(__int64 value, const RtlFieldInfo *field) arrow::ArrayBuilder *fieldBuilder = parquetWriter->getFieldBuilder(field); switch(fieldBuilder->type()->id()) { + case arrow::Type::type::INT8: + { + arrow::Int8Builder *int8Builder = static_cast(fieldBuilder); + reportIfFailure(int8Builder->Append(value)); + break; + } + case arrow::Type::type::INT16: + { + arrow::Int16Builder *int16Builder = static_cast(fieldBuilder); + reportIfFailure(int16Builder->Append(value)); + break; + } case arrow::Type::type::INT32: { arrow::Int32Builder *int32Builder = static_cast(fieldBuilder); @@ -1791,6 +1841,18 @@ void ParquetRecordBinder::processUInt(unsigned __int64 value, const RtlFieldInfo arrow::ArrayBuilder *fieldBuilder = parquetWriter->getFieldBuilder(field); switch(fieldBuilder->type()->id()) { + case arrow::Type::type::UINT8: + { + arrow::UInt8Builder *uInt8Builder = static_cast(fieldBuilder); + reportIfFailure(uInt8Builder->Append(value)); + break; + } + case arrow::Type::type::UINT16: + { + arrow::UInt16Builder *uInt16Builder = static_cast(fieldBuilder); + reportIfFailure(uInt16Builder->Append(value)); + break; + } case arrow::Type::type::UINT32: { arrow::UInt32Builder *uInt32Builder = static_cast(fieldBuilder); @@ -1822,6 +1884,11 @@ void ParquetRecordBinder::processReal(double value, const RtlFieldInfo *field) arrow::DoubleBuilder *doubleBuilder = static_cast(fieldBuilder); reportIfFailure(doubleBuilder->Append(value)); } + else if (fieldBuilder->type()->id() == arrow::Type::type::FLOAT) + { + arrow::FloatBuilder *floatBuilder = static_cast(fieldBuilder); + reportIfFailure(floatBuilder->Append(value)); + } else failx("Incorrect type for real field %s: %s", field->name, fieldBuilder->type()->ToString().c_str()); } From 14ad69bfac2011fac1db90cf144a461b1ebc78f0 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:22:05 -0400 Subject: [PATCH 10/23] HPCC-32585 ECL Watch v9 redirect security routes non-admin adds logic to the security/* routes that will prevent a non-admin user from reaching these urls Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/hooks/user.ts | 5 +++++ esp/src/src-react/routes.tsx | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/esp/src/src-react/hooks/user.ts b/esp/src/src-react/hooks/user.ts index a1b49dab49e..0eefa837e6c 100644 --- a/esp/src/src-react/hooks/user.ts +++ b/esp/src/src-react/hooks/user.ts @@ -3,6 +3,8 @@ import { useConst, useForceUpdate } from "@fluentui/react-hooks"; import { AccessService, AccountService, WsAccount } from "@hpcc-js/comms"; import { cookieKeyValStore } from "src/KeyValStore"; +declare const dojoConfig; + const defaults = { ESPSessionTimeout: 7200, ESPAuthenticated: false, @@ -97,12 +99,15 @@ export function useMyAccount(): { currentUser: WsAccount.MyAccountResponse, isAd const adminGroupNames = ["Administrator", "Directory Administrators"]; if (response.isLDAPAdmin || groups.filter(group => !adminGroupNames.indexOf(group.name)).length > 0) { setIsAdmin(true); + dojoConfig.isAdmin = true; } else { setIsAdmin(account.accountType === "Administrator"); + dojoConfig.isAdmin = account.accountType === "Administrator"; } }); } else { setIsAdmin(true); + dojoConfig.isAdmin = true; } setCurrentUser(account); }); diff --git a/esp/src/src-react/routes.tsx b/esp/src/src-react/routes.tsx index 39b52b822bc..bee1c306fd9 100644 --- a/esp/src/src-react/routes.tsx +++ b/esp/src/src-react/routes.tsx @@ -2,6 +2,8 @@ import * as React from "react"; import { Route, RouterContext } from "universal-router"; import { initialize, parsePage, parseSearch, parseSort, pushUrl, replaceUrl } from "./util/history"; +declare const dojoConfig; + export type MainNav = "activities" | "workunits" | "files" | "queries" | "topology" | "operations"; export interface RouteEx extends Route { @@ -327,6 +329,7 @@ export const routes: RoutesEx = [ }, { path: "/security", + action: () => { if (!dojoConfig.isAdmin) { replaceUrl("/topology"); } }, children: [ { path: "", action: (ctx, params) => import("./components/Security").then(_ => { @@ -462,6 +465,7 @@ export const routes: RoutesEx = [ { path: "/security", + action: () => { if (!dojoConfig.isAdmin) { replaceUrl("/operations"); } }, children: [ { path: "", action: (ctx, params) => import("./components/Security").then(_ => { From b95f56064c66bef50af54e3b931d0d9f7d8ad054 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 19 Aug 2024 14:55:41 +0100 Subject: [PATCH 11/23] HPCC-32458 Report unauthorized context details Signed-off-by: Jake Smith --- esp/bindings/SOAP/Platform/soapbind.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/bindings/SOAP/Platform/soapbind.cpp b/esp/bindings/SOAP/Platform/soapbind.cpp index b0e181a92bb..8e1b2ab9735 100644 --- a/esp/bindings/SOAP/Platform/soapbind.cpp +++ b/esp/bindings/SOAP/Platform/soapbind.cpp @@ -258,7 +258,7 @@ int CHttpSoapBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* re response->sendBasicChallenge(m_challenge_realm.str(), false); else if (status == SOAP_AUTHENTICATION_ERROR) { - throw MakeStringExceptionDirect(401,"Unauthorized Access"); + throw makeStringExceptionV(401, "Unauthorized Access: %s - user=%s", ctx->queryServiceName(nullptr), nullText(ctx->queryUserId())); } else response->setStatus(HTTP_STATUS_OK); From 7d0363dd8d3c1641191a3d2127b6f3a7500443ef Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 28 Aug 2024 16:07:30 +0100 Subject: [PATCH 12/23] HPCC-32543 Add statistic for number/size of agent requests Signed-off-by: Richard Chapman --- roxie/ccd/ccdserver.cpp | 11 +++++++---- system/jlib/jstatcodes.h | 2 ++ system/jlib/jstats.cpp | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 45a76675e46..58292a298c0 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -479,7 +479,7 @@ static const StatisticsMapping keyedJoinStatistics({ StNumServerCacheHits, StNum StCycleBlobFetchCycles, StCycleLeafFetchCycles, StCycleNodeFetchCycles, StTimeBlobFetch, StTimeLeafFetch, StTimeNodeFetch, StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked, StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches, - StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, + StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, StNumAgentRequests, StSizeAgentRequests, StSizeContinuationData, StNumContinuationRequests }, joinStatistics); static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks, StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares, @@ -491,10 +491,10 @@ static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndex StCycleBlobFetchCycles, StCycleLeafFetchCycles, StCycleNodeFetchCycles, StTimeBlobFetch, StTimeLeafFetch, StTimeNodeFetch, StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked, StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches, - StNumIndexRowsRead, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, + StNumIndexRowsRead, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, StNumAgentRequests, StSizeAgentRequests, StSizeContinuationData, StNumContinuationRequests }, actStatistics); static const StatisticsMapping diskStatistics({StNumServerCacheHits, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted, - StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, + StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, StNumAgentRequests, StSizeAgentRequests, StSizeContinuationData, StNumContinuationRequests }, actStatistics); static const StatisticsMapping soapStatistics({ StTimeSoapcall }, actStatistics); static const StatisticsMapping groupStatistics({ StNumGroups, StNumGroupMax }, actStatistics); @@ -526,7 +526,7 @@ extern const StatisticsMapping accumulatedStatistics({StWhenFirstRow, StTimeLoca StNumSocketWrites, StSizeSocketWrite, StTimeSocketWriteIO, StNumSocketReads, StSizeSocketRead, StTimeSocketReadIO, StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked, - StNumAckRetries, StSizeContinuationData, StNumContinuationRequests + StNumAckRetries, StNumAgentRequests, StSizeAgentRequests, StSizeContinuationData, StNumContinuationRequests }); //================================================================================= @@ -4255,7 +4255,10 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie byte * tgt = (byte*)(h+1); owner.copyHeader(tgt, channel); + owner.activity.noteStatistic(StNumAgentRequests, 1); + owner.activity.noteStatistic(StSizeAgentRequests, buffer.length()); ret.setown(createRoxiePacket(buffer)); + if (overflowSequence == OVERFLOWSEQUENCE_MAX) overflowSequence = 1; // don't wrap to 0 - that is a bit special else diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h index 9e034906601..1b40a879fe8 100644 --- a/system/jlib/jstatcodes.h +++ b/system/jlib/jstatcodes.h @@ -312,6 +312,8 @@ enum StatisticKind StNumMatchCandidates, StNumMatchCandidatesMax, StNumParallelExecute, + StNumAgentRequests, + StSizeAgentRequests, StMax, //For any quantity there is potentially the following variants. diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index dd0ad0b2034..45e0aa0e09f 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -984,6 +984,8 @@ static const constexpr StatisticMeta statsMetaData[StMax] = { { NUMSTAT(MatchCandidates), "The number of candidate combinations of left and right rows forming join groups" }, { NUMSTAT(MatchCandidatesMax), "The largest number of candidate combinations of left and right rows in a single group" }, { NUMSTAT(ParallelExecute), "The number of parallel execution paths for this activity" }, + { NUMSTAT(AgentRequests), "The number of agent request packets for this activity" }, + { SIZESTAT(AgentRequests), "The total size of agent request packets for this activity" }, }; static MapStringTo statisticNameMap(true); From 5e9ccd698e6c7c5d900052e0002c2f289632a9e1 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:33:04 -0400 Subject: [PATCH 13/23] HPCC-32591 ECL Watch v9 File scopes reduce Logical Name width reduces the minimum width of the Logical Name column in the files list when viewing by scopes Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Scopes.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/Scopes.tsx b/esp/src/src-react/components/Scopes.tsx index b03b33789b7..d0edefdc3b9 100644 --- a/esp/src/src-react/components/Scopes.tsx +++ b/esp/src/src-react/components/Scopes.tsx @@ -161,7 +161,7 @@ export const Scopes: React.FunctionComponent = ({ } }, Name: { - label: nlsHPCC.LogicalName, width: 600, + label: nlsHPCC.LogicalName, width: 180, formatter: (_, row) => { let name = row.Name?.split("::").pop(); let url = `#/files/${row.NodeGroup ? row.NodeGroup + "/" : ""}${[].concat(".", scopePath, name).join("::")}`; From bb6eafba1b6900104331f564b90dac406fd33ceb Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:33:57 -0400 Subject: [PATCH 14/23] HPCC-32586 ECL Watch v9 fix WU filter incorrect dates Fixes an issue with formatting dates submitted in filter query, where a datetime is reencoded to UTC even if that had already happened. Also corrects a problem on the Files list filter, where the EndDate was being assigned the StartDate's value. Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 4 +++- esp/src/src/ESPWorkunit.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 6cc33214cbb..95cf41ee7b6 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -70,10 +70,12 @@ function formatQuery(_filter): { [id: string]: any } { delete filter.NotInSuperFiles; delete filter.Indexes; if (filter.StartDate) { + if (filter.StartDate.indexOf("Z") < 0) { filter.StartDate += ":00.000Z"; } filter.StartDate = new Date(filter.StartDate).toISOString(); } if (filter.EndDate) { - filter.EndDate = new Date(filter.StartDate).toISOString(); + if (filter.EndDate.indexOf("Z") < 0) { filter.EndDate += ":00.000Z"; } + filter.EndDate = new Date(filter.EndDate).toISOString(); } return filter; } diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 0b7713df67f..1efadc5afc1 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -124,9 +124,11 @@ export function formatQuery(_filter): { [id: string]: any } { delete filter.LastNDays; } else { if (filter.StartDate) { + if (filter.StartDate.indexOf("Z") < 0) { filter.StartDate += ":00.000Z"; } filter.StartDate = new Date(filter.StartDate).toISOString(); } if (filter.EndDate) { + if (filter.EndDate.indexOf("Z") < 0) { filter.EndDate += ":00.000Z"; } filter.EndDate = new Date(filter.EndDate).toISOString(); } } From 29bbf61e7912092f9a7631448f243f24c14cb83d Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:00:16 -0400 Subject: [PATCH 15/23] HPCC-32597 ECL Watch v9 fix search results WU tab fixes an uncaught JS exception when viewing the WU tab of the global search results Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Workunits.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index ab47cf34673..5826e790dde 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -294,7 +294,7 @@ export const Workunits: React.FunctionComponent = ({ }, [selection]); const renderRowTimings = React.useCallback((props: IDetailsRowProps, size: { readonly width: number; readonly height: number; }) => { - if (showTimeline && props) { + if (showTimeline && props?.item?.timings) { const total = props.item.timings.page.end - props.item.timings.page.start; const startPct = 100 - (props.item.timings.start - props.item.timings.page.start) / total * 100; const endPct = 100 - (props.item.timings.end - props.item.timings.page.start) / total * 100; From 923ee1d47cb8bd64e67a77f165444e587dd57e26 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 5 Sep 2024 10:41:51 +0100 Subject: [PATCH 16/23] HPCC-32613 Fix follow on errors if CCompressedFile::close throws an exception Signed-off-by: Gavin Halliday --- system/jlib/jlzw.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 5629865855e..827223950b1 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2448,6 +2448,8 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface { CriticalBlock block(crit); if (mode!=ICFread) { + //Ensure that a subsequent close() does not re-enter this code if an exception is thrown + mode = ICFread; if (overflow.length()) { unsigned ol = overflow.length(); overflow.clear(); @@ -2471,7 +2473,6 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface if (fileio) fileio->close(); } - mode = ICFread; curblockpos = 0; curblocknum = (unsigned)-1; // relies on wrap } From 20e9379156eda6a8be15a8013633c4fd55a39620 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Tue, 27 Aug 2024 16:45:10 -0400 Subject: [PATCH 17/23] HPCC-32534 Improve logging when ensuring file access permissions Added addtional contextual information to existing exception messages Signed-Off-By: Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- dali/dfu/dfurun.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 160c51cf07e..3b470723a66 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -540,23 +540,31 @@ class CDFUengine: public CInterface, implements IDFUengine return result; } - void ensureFilePermissions(const char * planeName, const char * fileName, SecAccessFlags perm, bool write) + void ensureFilePermissions(const char * planeName, const char * fileName, SecAccessFlags perm, IUserDescriptor *user, bool write) { if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm))) { + StringBuffer context; + StringBuffer username; + if (user) + user->getUserName(username); + else + username.append("Null user"); + context.appendf("user: '%s', assigned access %s (%d)", username.str(), getSecAccessFlagName(perm), perm); + if (!isEmptyString(planeName)) { CDfsLogicalFileName dlfn; dlfn.setPlaneExternal(planeName, fileName); if (write) - throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for file scope: %s on DropZone: %s", dlfn.get(), planeName); + throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for file scope: %s on DropZone: %s, %s", dlfn.get(), planeName, context.str()); else - throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for file scope: %s on DropZone: %s", dlfn.get(), planeName); + throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for file scope: %s on DropZone: %s, %s", dlfn.get(), planeName, context.str()); } if (write) - throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s", fileName); + throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s, %s", fileName, context.str()); else - throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s", fileName); + throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s, %s", fileName, context.str()); } } @@ -618,12 +626,13 @@ class CDFUengine: public CInterface, implements IDFUengine void checkPhysicalFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write) { unsigned auditflags = (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED); + logNullUser(user);//stack trace if NULL user if (write) auditflags |= DALI_LDAP_WRITE_WANTED; SecAccessFlags perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); StringBuffer name; - ensureFilePermissions(nullptr,getFDescName(fd,name),perm,write); + ensureFilePermissions(nullptr,getFDescName(fd,name),perm,user,write); } void checkForeignFilePermissions(IConstDFUfileSpec *fSpec,IFileDescriptor *fd,IUserDescriptor *user) @@ -667,6 +676,7 @@ class CDFUengine: public CInterface, implements IDFUengine void checkPlaneFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write) { + logNullUser(user);//stack trace if NULL user //This function checks the scope permissions for a file or files that reside in a single directory on a single plane. //The IFileDescriptor is used to discover the plane and directory. //If the plane is not present, it implies that it is a bare-metal system and useDropZoneRestriction is off, and there @@ -697,7 +707,8 @@ class CDFUengine: public CInterface, implements IDFUengine { if (getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized())) perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); - ensureFilePermissions(planeName,relativePath,perm,write); + + ensureFilePermissions(planeName,relativePath,perm,user,write); } } else From 1d47e36af28649ec3adb46f0baf9639e43dcea68 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 5 Sep 2024 11:27:52 +0100 Subject: [PATCH 18/23] HPCC-32620 Include the filename when reporting a file error Signed-off-by: Gavin Halliday --- dali/datest/datest.cpp | 2 +- system/jlib/jfile.cpp | 46 +++++++++++++++++++++--------------------- system/jlib/jfile.hpp | 2 +- system/jlib/jfile.ipp | 9 ++++++++- system/jlib/jio.cpp | 32 ++++++++++++++--------------- system/jlib/jio.hpp | 6 +++--- system/jlib/jutil.cpp | 2 +- 7 files changed, 53 insertions(+), 46 deletions(-) diff --git a/dali/datest/datest.cpp b/dali/datest/datest.cpp index 88b23181d3e..84db25fe817 100644 --- a/dali/datest/datest.cpp +++ b/dali/datest/datest.cpp @@ -2927,7 +2927,7 @@ NULL #else out = fileno(stdout); #endif - Owned stdOutFileIO = createIFileIO(out,IFOwrite); + Owned stdOutFileIO = createIFileIO(nullptr,out,IFOwrite); if (testParams.ordinality()) { newFileName = testParams.item(0); diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 115a67c665e..de7853eb092 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -1785,8 +1785,8 @@ class jlib_decl CSequentialFileIO : public CFileIO } public: - CSequentialFileIO(HANDLE h,IFOmode _openmode,IFSHmode _sharemode,IFEflags _extraFlags) - : CFileIO(h,_openmode,_sharemode,_extraFlags) + CSequentialFileIO(IFile * _creator, HANDLE h,IFOmode _openmode,IFSHmode _sharemode,IFEflags _extraFlags) + : CFileIO(_creator, h,_openmode,_sharemode,_extraFlags) { pos = 0; } @@ -1808,7 +1808,7 @@ class jlib_decl CSequentialFileIO : public CFileIO } size32_t ret = (size32_t)numRead; #else - size32_t ret = checked_read(file, data, len); + size32_t ret = checked_read(querySafeFilename(), file, data, len); #endif pos += ret; return ret; @@ -1854,9 +1854,9 @@ IFileIO * CFile::openShared(IFOmode mode,IFSHmode share,IFEflags extraFlags) // No - while read-ahead can put more into page-cache, IFEnocache is a hint and // disabling readahead might affect performance negatively too much ... if (stdh>=0) - return new CSequentialFileIO(handle,mode,share,extraFlags); + return new CSequentialFileIO(this, handle,mode,share,extraFlags); - Owned io = new CFileIO(handle,mode,share,extraFlags); + Owned io = new CFileIO(this,handle,mode,share,extraFlags); #ifdef CHECK_FILE_IO return new CCheckingFileIO(filename, io); #else @@ -1869,9 +1869,9 @@ IFileIO * CFile::openShared(IFOmode mode,IFSHmode share,IFEflags extraFlags) //--------------------------------------------------------------------------- -extern jlib_decl IFileIO *createIFileIO(HANDLE handle,IFOmode openmode,IFEflags extraFlags) +extern jlib_decl IFileIO *createIFileIO(IFile * creator,HANDLE handle,IFOmode openmode,IFEflags extraFlags) { - return new CFileIO(handle,openmode,IFSHfull,extraFlags); + return new CFileIO(creator, handle,openmode,IFSHfull,extraFlags); } offset_t CFileIO::appendFile(IFile *file,offset_t pos,offset_t len) @@ -1910,8 +1910,8 @@ unsigned __int64 CFileIO::getStatistic(StatisticKind kind) //-- Windows implementation ------------------------------------------------- -CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags) - : unflushedReadBytes(0), unflushedWriteBytes(0) +CFileIO::CFileIO(IFile * _creator, HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags) + : creator(_creator), unflushedReadBytes(0), unflushedWriteBytes(0) { assertex(handle != NULLFILE); throwOnError = false; @@ -1948,14 +1948,14 @@ void CFileIO::close() std::swap(tmpHandle, file); if (!CloseHandle(tmpHandle)) - throw makeOsException(GetLastError(),"CFileIO::close"); + throw makeOsExceptionV(GetLastError(),"CFileIO::close for file '%s'", querySafeFilename()); } } void CFileIO::flush() { if (!FlushFileBuffers(file)) - throw makeOsException(GetLastError(),"CFileIO::flush"); + throw makeOsExceptionV(GetLastError(),"CFileIO::flush for file '%s'", querySafeFilename()); } offset_t CFileIO::size() @@ -1979,7 +1979,7 @@ size32_t CFileIO::read(offset_t pos, size32_t len, void * data) DWORD numRead; setPos(pos); if (ReadFile(file,data,len,&numRead,NULL) == 0) - throw makeOsException(GetLastError(),"CFileIO::read"); + throw makeOsExceptionV(GetLastError(),"CFileIO::read for file '%s'", querySafeFilename()); stats.ioReadCycles.fetch_add(timer.elapsedCycles()); stats.ioReadBytes.fetch_add(numRead); ++stats.ioReads; @@ -2001,9 +2001,9 @@ size32_t CFileIO::write(offset_t pos, size32_t len, const void * data) DWORD numWritten; setPos(pos); if (!WriteFile(file,data,len,&numWritten,NULL)) - throw makeOsException(GetLastError(),"CFileIO::write"); + throw makeOsExceptionV(GetLastError(),"CFileIO::write for file '%s'", querySafeFilename()); if (numWritten != len) - throw makeOsException(DISK_FULL_EXCEPTION_CODE,"CFileIO::write"); + throw makeOsExceptionV(DISK_FULL_EXCEPTION_CODE,"CFileIO::write for file '%s'", querySafeFilename()); stats.ioWriteCycles.fetch_add(timer.elapsedCycles()); stats.ioWriteBytes.fetch_add(numWritten); ++stats.ioWrites; @@ -2016,7 +2016,7 @@ void CFileIO::setSize(offset_t pos) CriticalBlock procedure(cs); setPos(pos); if (!SetEndOfFile(file)) - throw makeOsException(GetLastError(), "CFileIO::setSize"); + throw makeOsExceptionV(GetLastError(), "CFileIO::setSize for file '%s'", querySafeFilename()); } #else @@ -2053,8 +2053,8 @@ static void syncFileData(int fd, bool notReadOnly, IFEflags extraFlags, bool wai } // More errorno checking TBD -CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags) - : unflushedReadBytes(0), unflushedWriteBytes(0) +CFileIO::CFileIO(IFile * _creator, HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags _extraFlags) + : creator(_creator), unflushedReadBytes(0), unflushedWriteBytes(0) { assertex(handle != NULLFILE); throwOnError = false; @@ -2082,7 +2082,7 @@ CFileIO::~CFileIO() //Any file that is being written to, should be closed before the object is destroyed, otherwise errors from failing to commit will be lost if ((file != NULLFILE) && (openmode!=IFOread)) { - OERRLOG("CFileIO::~CFileIO - file object destroyed without being closed first"); // A programmer problem, but the operator should know about it. + OERRLOG("CFileIO::~CFileIO - file '%s' object destroyed without being closed first", querySafeFilename()); // A programmer problem, but the operator should know about it. PrintStackReport(); } #endif @@ -2114,7 +2114,7 @@ void CFileIO::close() syncFileData(tmpHandle, openmode!=IFOread, extraFlags, false); if (::close(tmpHandle) < 0) - throw makeErrnoException(errno, "CFileIO::close"); + throw makeErrnoExceptionV(errno, "CFileIO::close for file '%s'", querySafeFilename()); } } @@ -2152,7 +2152,7 @@ size32_t CFileIO::read(offset_t pos, size32_t len, void * data) if (0==len) return 0; CCycleTimer timer; - size32_t ret = checked_pread(file, data, len, pos); + size32_t ret = checked_pread(querySafeFilename(), file, data, len, pos); stats.ioReadCycles.fetch_add(timer.elapsedCycles()); stats.ioReadBytes.fetch_add(ret); ++stats.ioReads; @@ -2183,9 +2183,9 @@ size32_t CFileIO::write(offset_t pos, size32_t len, const void * data) ++stats.ioWrites; if (ret==(size32_t)-1) - throw makeErrnoException(errno, "CFileIO::write"); + throw makeErrnoExceptionV(errno, "CFileIO::write for file '%s'", querySafeFilename()); if (ret 0) ) { if (unflushedWriteBytes.add_fetch(ret) >= PGCFLUSH_BLKSIZE) @@ -2201,7 +2201,7 @@ size32_t CFileIO::write(offset_t pos, size32_t len, const void * data) void CFileIO::setSize(offset_t pos) { if (0 != ftruncate(file, pos)) - throw makeErrnoException(errno, "CFileIO::setSize"); + throw makeErrnoExceptionV(errno, "CFileIO::setSize for file '%s'", querySafeFilename()); } #endif diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index c55c5b7ccdd..fa2a27b4275 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -274,7 +274,7 @@ extern jlib_decl IFile * createIFile(const char * filename); extern jlib_decl IFile * createIFile(MemoryBuffer & buffer); extern jlib_decl void touchFile(const char *filename); extern jlib_decl void touchFile(IFile *file); -extern jlib_decl IFileIO * createIFileIO(HANDLE handle,IFOmode mode,IFEflags extraFlags=IFEnone); +extern jlib_decl IFileIO * createIFileIO(IFile * creator, HANDLE handle,IFOmode mode,IFEflags extraFlags=IFEnone); extern jlib_decl IDirectoryIterator * createDirectoryIterator(const char * path = NULL, const char * wildcard = NULL, bool sub = false, bool includedirs = true); extern jlib_decl IDirectoryIterator * createNullDirectoryIterator(); extern jlib_decl IFileIO * createIORange(IFileIO * file, offset_t header, offset_t length); // restricts input/output to a section of a file. diff --git a/system/jlib/jfile.ipp b/system/jlib/jfile.ipp index 56682a53bcf..815e8163bf9 100644 --- a/system/jlib/jfile.ipp +++ b/system/jlib/jfile.ipp @@ -91,7 +91,7 @@ protected: class jlib_decl CFileIO : implements IFileIO, public CInterface { public: - CFileIO(HANDLE,IFOmode _openmode,IFSHmode _sharemode,IFEflags _extraFlags); + CFileIO(IFile * _creator,HANDLE,IFOmode _openmode,IFSHmode _sharemode,IFEflags _extraFlags); ~CFileIO(); IMPLEMENT_IINTERFACE @@ -105,9 +105,16 @@ public: virtual unsigned __int64 getStatistic(StatisticKind kind); HANDLE queryHandle() { return file; } // for debugging + const char * queryFilename() const { return creator ? creator->queryFilename() : nullptr; } + const char * querySafeFilename() const + { + const char * name = queryFilename(); + return name ? name : ""; + } protected: CriticalSection cs; + Linked creator; HANDLE file; bool throwOnError; IFSHmode sharemode; diff --git a/system/jlib/jio.cpp b/system/jlib/jio.cpp index 649e092a7e5..8a51f4e2176 100644 --- a/system/jlib/jio.cpp +++ b/system/jlib/jio.cpp @@ -56,7 +56,7 @@ extern jlib_decl offset_t checked_lseeki64( int handle, offset_t offset, int ori return ret; } -extern jlib_decl size32_t checked_read(int file, void *buffer, size32_t len) +extern jlib_decl size32_t checked_read(const char * filename, int file, void *buffer, size32_t len) { if (0==len) return 0; unsigned attempts = 0; @@ -86,7 +86,7 @@ extern jlib_decl size32_t checked_read(int file, void *buffer, size32_t len) readNow = 0; break; } - throw makeErrnoException(errno, "checked_read"); + throw makeErrnoExceptionV(errno, "checked_read for file '%s'", filename); } } else if (!readNow) @@ -105,7 +105,7 @@ static bool atomicsupported = true; static CriticalSection atomicsection; #endif -extern jlib_decl size32_t checked_pread(int file, void *buffer, size32_t len, offset_t pos) +extern jlib_decl size32_t checked_pread(const char * filename, int file, void *buffer, size32_t len, offset_t pos) { if (0==len) return 0; #ifdef WIN32 @@ -125,12 +125,12 @@ extern jlib_decl size32_t checked_pread(int file, void *buffer, size32_t len, of if (err == ERROR_INVALID_PARAMETER) // Win98 etc atomicsupported = false; else - throw makeOsException(GetLastError(), "checked_pread"); + throw makeOsExceptionV(GetLastError(), "checked_pread for file '%s'", filename); } { CriticalBlock blk(atomicsection); checked_lseeki64(file, pos, FILE_BEGIN); - return checked_read(file, buffer, len); + return checked_read(filename, file, buffer, len); } #else size32_t ret = 0; @@ -160,7 +160,7 @@ extern jlib_decl size32_t checked_pread(int file, void *buffer, size32_t len, of readNow = 0; break; } - throw makeErrnoException(errno, "checked_pread"); + throw makeErrnoExceptionV(errno, "checked_pread for file '%s'", filename); } } else if (!readNow) @@ -176,15 +176,15 @@ extern jlib_decl size32_t checked_pread(int file, void *buffer, size32_t len, of #endif } -extern jlib_decl size32_t checked_write( int handle, const void *buffer, size32_t count ) +extern jlib_decl size32_t checked_write(const char * filename, int handle, const void *buffer, size32_t count ) { int ret=_write(handle,buffer,count); if ((size32_t)ret != count) { if (-1 != ret) - throw makeOsException(DISK_FULL_EXCEPTION_CODE, "checked_write"); + throw makeOsExceptionV(DISK_FULL_EXCEPTION_CODE, "checked_write for file '%s'", filename); else - throw makeErrnoException(errno, "checked_write"); + throw makeErrnoExceptionV(errno, "checked_write for file '%s'", filename); } return (size32_t)ret; } @@ -241,7 +241,7 @@ class CReadSeq : public IReadSeq, public CInterface if (endpos-nextbufpos<(offset_t)rd) rd = (size32_t)(endpos-nextbufpos); if (rd) - rd = checked_pread(fh, buffer+left, rd, nextbufpos); + rd = checked_pread("unknown", fh, buffer+left, rd, nextbufpos); nextbufpos += rd; bytesInBuffer = left+rd; ptr = buffer; @@ -389,7 +389,7 @@ class CWriteSeq : public IWriteSeq, public CInterface } while (_size>=bufSize) // write out directly { - checked_write(fh, src, bufSize); // stick to writing bufSize blocks + checked_write("unknown", fh, src, bufSize); // stick to writing bufSize blocks src = (char *)src + bufSize; _size -= bufSize; } @@ -470,7 +470,7 @@ class CWriteSeq : public IWriteSeq, public CInterface { if (ptr != buffer) { - checked_write(fh, buffer, (size32_t)(ptr-buffer)); + checked_write("unknown", fh, buffer, (size32_t)(ptr-buffer)); ptr = buffer; } } @@ -514,13 +514,13 @@ CUnbufferedReadWriteSeq::CUnbufferedReadWriteSeq(int _fh, offset_t _offset, size void CUnbufferedReadWriteSeq::put(const void *src) { - checked_write(fh, src, size); + checked_write("unknown", fh, src, size); fpos += size; } void CUnbufferedReadWriteSeq::putn(const void *src, unsigned n) { - checked_write(fh, src, size*n); + checked_write("unknown", fh, src, size*n); fpos += size*n; } @@ -537,7 +537,7 @@ bool CUnbufferedReadWriteSeq::get(void *dst) size32_t toread = size; while (toread) { - int read = checked_read(fh, dst, toread); + int read = checked_read("unknown", fh, dst, toread); if (!read) return false; toread -= read; @@ -552,7 +552,7 @@ unsigned CUnbufferedReadWriteSeq::getn(void *dst, unsigned n) size32_t totread = 0; while (toread) { - int read = checked_read(fh, dst, toread); + int read = checked_read("unknown", fh, dst, toread); if (!read) break; toread -= read; diff --git a/system/jlib/jio.hpp b/system/jlib/jio.hpp index 2c3af787591..99ff19041e0 100644 --- a/system/jlib/jio.hpp +++ b/system/jlib/jio.hpp @@ -135,9 +135,9 @@ extern jlib_decl unsigned copySeq(IReadSeq *from,IWriteSeq *to,size32_t bufsize) extern jlib_decl void setIORetryCount(unsigned _ioRetryCount); // default 0 == off, retries if read op. fails extern jlib_decl offset_t checked_lseeki64(int handle, offset_t offset, int origin); -extern jlib_decl size32_t checked_write(int handle, const void *buffer, size32_t count); -extern jlib_decl size32_t checked_read(int file, void *buffer, size32_t len); -extern jlib_decl size32_t checked_pread(int file, void *buffer, size32_t len, offset_t pos); +extern jlib_decl size32_t checked_write(const char * filename, int handle, const void *buffer, size32_t count); +extern jlib_decl size32_t checked_read(const char * filename, int file, void *buffer, size32_t len); +extern jlib_decl size32_t checked_pread(const char * filename, int file, void *buffer, size32_t len, offset_t pos); interface IFileIO; interface IFileIOStream; diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp index 109945c56c6..7493611f88a 100644 --- a/system/jlib/jutil.cpp +++ b/system/jlib/jutil.cpp @@ -3432,7 +3432,7 @@ void jlib_decl atomicWriteFile(const char *fileName, const char *output) Owned newFile = createIFile(newFileName); Owned file = createIFile(fileName); { - OwnedIFileIO ifileio = createIFileIO(fh, IFOwrite); + OwnedIFileIO ifileio = createIFileIO(file, fh, IFOwrite); if (!ifileio) throw MakeStringException(0, "atomicWriteFile: could not create output file %s", newFileName.str()); ifileio->write(0, strlen(output), output); From 564a0302b228e4d69ada5b726638b82b816c83db Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 2 Aug 2024 16:32:52 +0100 Subject: [PATCH 19/23] HPCC-32244 Add custom views to Metrics Signed-off-by: Gordon Smith --- esp/src/package-lock.json | 18 + esp/src/package.json | 1 + esp/src/src-react/components/Metrics.tsx | 90 +++-- .../src-react/components/MetricsOptions.tsx | 349 +++++++++++++----- .../src-react/components/controls/Grid.tsx | 53 ++- esp/src/src-react/hooks/metrics.ts | 222 ++++++++--- esp/src/src-react/util/metricGraph.ts | 14 +- 7 files changed, 546 insertions(+), 201 deletions(-) diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index ecb1390c775..3391efbb526 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -51,6 +51,7 @@ "react-hook-form": "7.51.2", "react-hot-toast": "2.4.1", "react-reflex": "4.2.6", + "react-singleton-hook": "3.4.0", "react-sizeme": "3.0.2", "universal-router": "9.2.0", "xstyle": "0.3.3" @@ -9777,6 +9778,23 @@ "react-dom": ">0.13.0" } }, + "node_modules/react-singleton-hook": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/react-singleton-hook/-/react-singleton-hook-3.4.0.tgz", + "integrity": "sha512-eQEpyacGAaRejmWUizUdNNQFn5AO0iaKRSl1jxgC0FQadVY/I1WFuPrYiutglPzO9s8yEbIh95UXVJQel4d7HQ==", + "license": "MIT", + "peerDependencies": { + "react": "15 - 18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + }, + "react-native": { + "optional": true + } + } + }, "node_modules/react-sizeme": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/react-sizeme/-/react-sizeme-3.0.2.tgz", diff --git a/esp/src/package.json b/esp/src/package.json index e9d7224d6f4..de3511485ec 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -77,6 +77,7 @@ "react-hook-form": "7.51.2", "react-hot-toast": "2.4.1", "react-reflex": "4.2.6", + "react-singleton-hook": "3.4.0", "react-sizeme": "3.0.2", "universal-router": "9.2.0", "xstyle": "0.3.3" diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index 40413d012c7..95d16528912 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -11,7 +11,7 @@ import { scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; import { WUTimelineNoFetch } from "src/Timings"; import * as Utility from "src/Utility"; -import { FetchStatus, useMetricsOptions, useWUQueryMetrics, MetricsOptions as MetricsOptionsT } from "../hooks/metrics"; +import { FetchStatus, useMetricsViews, useWUQueryMetrics } from "../hooks/metrics"; import { HolyGrail } from "../layouts/HolyGrail"; import { AutosizeComponent, AutosizeHpccJSComponent } from "../layouts/HpccJSAdapter"; import { DockPanel, DockPanelItem, ResetableDockPanel } from "../layouts/DockPanel"; @@ -89,10 +89,10 @@ class TableEx extends Table { } _rawDataMap: { [id: number]: string } = {}; - metrics(metrics: any[], options: MetricsOptionsT, scopeFilter: string, matchCase: boolean): this { + metrics(metrics: any[], scopeTypes: string[], properties: string[], scopeFilter: string, matchCase: boolean): this { this .columns(["##"]) // Reset hash to force recalculation of default widths - .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...options.properties, "__StdDevs"]) + .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...properties, "__StdDevs"]) .columnFormats([ new ColumnFormatEx() .column("StdDevs") @@ -106,18 +106,18 @@ class TableEx extends Table { .data(metrics .filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase)) .filter(row => { - return options.scopeTypes.indexOf(row.type) >= 0; + return scopeTypes.indexOf(row.type) >= 0; }).map((row, idx) => { if (idx === 0) { this._rawDataMap = { 0: "##", 1: "type", 2: "__StdDevs", 3: "name" }; - options.properties.forEach((p, idx2) => { + properties.forEach((p, idx2) => { this._rawDataMap[4 + idx2] = p; }); } row.__hpcc_id = row.name; - return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...options.properties.map(p => { + return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...properties.map(p => { return row.__groupedProps[p]?.Value ?? row.__groupedProps[p]?.Max ?? row.__groupedProps[p]?.Avg ?? @@ -157,6 +157,7 @@ class TableEx extends Table { } type SelectedMetricsSource = "" | "scopesTable" | "scopesSqlTable" | "metricGraphWidget" | "hotspot" | "reset"; +const TIMELINE_FIXEDHEIGHT = 152; interface MetricsProps { wuid: string; @@ -178,10 +179,9 @@ export const Metrics: React.FunctionComponent = ({ const [selectedMetrics, setSelectedMetrics] = React.useState([]); const [selectedMetricsPtr, setSelectedMetricsPtr] = React.useState(-1); const [metrics, columns, _activities, _properties, _measures, _scopeTypes, fetchStatus, refresh] = useWUQueryMetrics(wuid, querySet, queryId); + const { viewIds, viewId, setViewId, view, updateView } = useMetricsViews(); const [showMetricOptions, setShowMetricOptions] = React.useState(false); - const [options, setOptions, saveOptions] = useMetricsOptions(); const [dockpanel, setDockpanel] = React.useState(); - const [showTimeline, setShowTimeline] = React.useState(true); const [trackSelection, setTrackSelection] = React.useState(true); const [fullscreen, setFullscreen] = React.useState(false); const [hotspots, setHotspots] = React.useState(""); @@ -240,11 +240,14 @@ export const Metrics: React.FunctionComponent = ({ }, [parentUrl, timeline]); React.useEffect(() => { - timeline - .scopes(metrics) - .lazyRender() - ; - }, [metrics, timeline]); + if (view.showTimeline) { + timeline + .scopes(metrics) + .height(TIMELINE_FIXEDHEIGHT) + .lazyRender() + ; + } + }, [metrics, timeline, view.showTimeline]); // Scopes Table --- const onChangeScopeFilter = React.useCallback((event: React.FormEvent, newValue?: string) => { @@ -258,7 +261,7 @@ export const Metrics: React.FunctionComponent = ({ const scopesTable = useConst(() => new TableEx() .multiSelect(true) - .metrics([], options, scopeFilter, matchCase) + .metrics([], view.scopeTypes, view.properties, scopeFilter, matchCase) .sortable(true) ); @@ -274,13 +277,13 @@ export const Metrics: React.FunctionComponent = ({ React.useEffect(() => { scopesTable - .metrics(metrics, options, scopeFilter, matchCase) + .metrics(metrics, view.scopeTypes, view.properties, scopeFilter, matchCase) .lazyRender() ; - }, [matchCase, metrics, options, scopeFilter, scopesTable]); + }, [matchCase, metrics, scopeFilter, scopesTable, view.properties, view.scopeTypes]); const updateScopesTable = React.useCallback((selection: IScope[]) => { - if (scopesTable?.renderCount() > 0) { + if (scopesTable?.renderCount() > 0 && selectedMetricsSource !== "scopesTable") { scopesTable.selection([]); if (selection.length) { const selRows = scopesTable.data().filter(row => { @@ -291,7 +294,7 @@ export const Metrics: React.FunctionComponent = ({ }); } } - }, [scopesTable]); + }, [scopesTable, selectedMetricsSource]); // Graph --- const metricGraph = useConst(() => new MetricGraph()); @@ -490,9 +493,9 @@ export const Metrics: React.FunctionComponent = ({ }, [crossTabTable]); React.useEffect(() => { - const dot = metricGraph.graphTpl(selectedLineage ? [selectedLineage] : [], options); + const dot = metricGraph.graphTpl(selectedLineage ? [selectedLineage] : [], view); setDot(dot); - }, [metricGraph, options, selectedLineage]); + }, [metricGraph, view, selectedLineage]); React.useEffect(() => { let cancelled = false; @@ -530,13 +533,14 @@ export const Metrics: React.FunctionComponent = ({ React.useEffect(() => { // Update layout prior to unmount --- - if (dockpanel && options && saveOptions && setOptions) { + if (dockpanel && updateView) { return () => { - setOptions({ ...options, layout: dockpanel.getLayout() }); - saveOptions(); + if (dockpanel && updateView) { + updateView({ layout: dockpanel.getLayout() }); + } }; } - }, [dockpanel, options, saveOptions, setOptions]); + }, [dockpanel, updateView]); // Command Bar --- const buttons = React.useMemo((): ICommandBarItemProps[] => [ @@ -556,19 +560,30 @@ export const Metrics: React.FunctionComponent = ({ }, { key: "divider_1", itemType: ContextualMenuItemType.Divider, onRender: () => }, { - key: "timeline", text: nlsHPCC.Timeline, canCheck: true, checked: showTimeline, iconProps: { iconName: "TimelineProgress" }, + key: "views", text: viewId, iconProps: { iconName: "View" }, + subMenuProps: { + items: viewIds.map(v => ({ + key: v, text: v, onClick: () => { + updateView({ layout: dockpanel.getLayout() }); + setViewId(v); + } + })) + }, + }, + { + key: "timeline", text: nlsHPCC.Timeline, canCheck: true, checked: view.showTimeline, iconProps: { iconName: "TimelineProgress" }, onClick: () => { - setShowTimeline(!showTimeline); + updateView({ showTimeline: !view.showTimeline }, true); } }, { key: "options", text: nlsHPCC.Options, iconProps: { iconName: "Settings" }, onClick: () => { - setOptions({ ...options, layout: dockpanel.layout() }); + updateView({ layout: dockpanel.getLayout() }); setShowMetricOptions(true); } } - ], [dockpanel, hotspots, onHotspot, options, refresh, setOptions, showTimeline, timeline]); + ], [dockpanel, hotspots, onHotspot, refresh, setViewId, timeline, updateView, view.showTimeline, viewId, viewIds]); const formatColumns = React.useMemo((): Utility.ColumnMap => { const copyColumns: Utility.ColumnMap = {}; @@ -610,7 +625,9 @@ export const Metrics: React.FunctionComponent = ({ } }] } - }, { + }, + { key: "divider_2", itemType: ContextualMenuItemType.Divider, onRender: () => }, + { key: "fullscreen", title: nlsHPCC.MaximizeRestore, iconProps: { iconName: fullscreen ? "ChromeRestore" : "FullScreen" }, onClick: () => setFullscreen(!fullscreen) } @@ -618,23 +635,18 @@ export const Metrics: React.FunctionComponent = ({ const setShowMetricOptionsHook = React.useCallback((show: boolean) => { setShowMetricOptions(show); - scopesTable - .metrics(metrics, options, scopeFilter, matchCase) - .render(() => { - updateScopesTable(selectedMetrics); - }) - ; + }, []); - }, [matchCase, metrics, options, scopeFilter, scopesTable, selectedMetrics, updateScopesTable]); + console.log("View ID", viewId, view.scopeTypes); return -