Jobstats

common/thorhelper/thorcommon.hpp

@@ -682,7 +682,10 @@ class CStatsContextLogger : public CSimpleInterfaceOf<IContextLogger>
     mutable CRuntimeStatisticCollection stats;
 public:
     CStatsContextLogger(const CRuntimeStatisticCollection  &_mapping, const LogMsgJobInfo & _job=unknownJob) : job(_job), stats(_mapping) {}
-
+    void reset()
+    {
+        stats.reset();
+    }
     virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override  __attribute__((format(printf,5,0)))
     {
         VALOG(cat, job, code, format, args);

ecl/eclagent/eclagent.cpp

@@ -576,6 +576,16 @@ EclAgent::EclAgent(IConstWorkUnit *wu, const char *_wuid, bool _checkVersion, bo
             abortmonitor->setGuillotineCost(money2cost_type(guillotineCost));
         }
     }
+
+    unsigned keyNodeCacheMB = wu->getDebugValueInt("keyNodeCacheMB", 100);
+    unsigned keyLeafCacheMB = wu->getDebugValueInt("keyLeafCacheMB", 50);
+    unsigned keyBlobCacheMB = wu->getDebugValueInt("keyBlobCacheMB", 0);
+    setNodeCacheMem(((memsize_t)0x100000) * keyNodeCacheMB);
+    setLeafCacheMem(((memsize_t)0x100000) * keyLeafCacheMB);
+    setBlobCacheMem(((memsize_t)0x100000) * keyBlobCacheMB);
+    if (wu->hasDebugValue("nodeFetchThresholdNs"))
+        setNodeFetchThresholdNs(wu->getDebugValueInt64("nodeFetchThresholdNs", 0));
+
     configurePreferredPlanes();
 }
 

system/jhtree/jhtree.cpp

@@ -572,10 +572,13 @@ class jhtree_decl CKeyLevelManager : implements IKeyManager, public CInterface
         filter->describe(out);
     }
 
-    virtual void mergeStats(CRuntimeStatisticCollection & stats) const
+    virtual void mergeStats(CRuntimeStatisticCollection & targetStats) const
     {
+        // IO Stats comming from the keyCursor and jhtree cache stats coming from this class's stats
         if (keyCursor)
-            keyCursor->mergeStats(stats);
+            keyCursor->mergeStats(targetStats); // merge IO stats
+        if (ctx)
+            targetStats.merge(ctx->queryStats()); // merge jhtree cache stats
     }
 };
 

system/jlib/jstats.cpp

@@ -1322,8 +1322,8 @@ const StatisticsMapping noStatistics({});
 const StatisticsMapping jhtreeCacheStatistics({ StNumIndexSeeks, StNumIndexScans, StNumPostFiltered, StNumIndexWildSeeks,
                                                 StNumNodeCacheAdds, StNumLeafCacheAdds, StNumBlobCacheAdds, StNumNodeCacheHits, StNumLeafCacheHits, StNumBlobCacheHits, StCycleNodeLoadCycles, StCycleLeafLoadCycles,
                                                 StCycleBlobLoadCycles, StCycleNodeReadCycles, StCycleLeafReadCycles, StCycleBlobReadCycles, StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches,
-                                                StCycleNodeFetchCycles, StCycleLeafFetchCycles, StCycleBlobFetchCycles, StCycleIndexCacheBlockedCycles, StNumIndexMergeCompares, StNumIndexMerges, StNumIndexSkips, StNumIndexNullSkips,
-                                                StTimeLeafLoad, StTimeLeafRead, StTimeLeafFetch, StTimeIndexCacheBlocked, StTimeNodeFetch});
+                                                StCycleNodeFetchCycles, StCycleLeafFetchCycles, StCycleBlobFetchCycles, StCycleIndexCacheBlockedCycles, StNumIndexMergeCompares, StNumIndexMerges, StNumIndexSkips,
+                                                StNumIndexNullSkips, StTimeLeafLoad, StTimeLeafRead, StTimeLeafFetch, StTimeIndexCacheBlocked, StTimeNodeFetch, StTimeNodeLoad, StTimeNodeRead});
 
 const StatisticsMapping allStatistics(StKindAll);
 const StatisticsMapping heapStatistics({StNumAllocations, StNumAllocationScans});

testing/regress/ecl/common/jobstats.ecl

@@ -0,0 +1,75 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+EXPORT JobStats := MODULE
+
+  import ^ as root;
+  import Std.File;
+  import lib_WorkunitServices.WorkunitServices;
+  import $.^.setup;
+
+  Files := Setup.Files(true, false);
+  SHARED wordIndex := Files.getWordIndex();
+
+  EXPORT jobStatKindRec := RECORD
+   string stat;
+  END;
+
+  EXPORT getStats(string wuid, string filter, DATASET(jobStatKindRec) statKinds) := FUNCTION
+    stats := NOTHOR(WorkunitServices.WorkunitStatistics(wuid, false, filter));
+    j := JOIN(stats, statKinds, LEFT.name = RIGHT.stat);
+    s := SORT(j, name);
+    RETURN TABLE(s, {name, value});
+  END;
+
+  SHARED searchWords := TABLE(wordIndex, {word}) : INDEPENDENT; // so that in a different workflow item from kj, to isolate jhtree cache usage
+
+  EXPORT TestKJSimple(boolean forceRemote = false, boolean forceLocal = false) := FUNCTION
+    RETURN IF(forceRemote,
+             JOIN(searchWords, wordIndex, RIGHT.kind = 1 AND LEFT.word = RIGHT.word, HINT(forceRemoteKeyedLookup(true))),
+             IF(forceLocal,
+                JOIN(searchWords, wordIndex, RIGHT.kind = 1 AND LEFT.word = RIGHT.word, HINT(remoteKeyedLookup(false))),
+                JOIN(searchWords, wordIndex, RIGHT.kind = 1 AND LEFT.word = RIGHT.word)
+               )
+            );
+  END;
+
+  EXPORT TestKJCQ(boolean forceRemote = false, boolean forceLocal = false) := FUNCTION
+    rec := RECORD
+      unsigned1 kind;
+      unsigned1 wip;
+      unsigned8 wPosSum := 0;
+    END;
+
+    rec cqtrans(rec l) := TRANSFORM
+      kj := IF(forceRemote,
+               JOIN(searchWords, wordIndex, RIGHT.kind = l.kind AND LEFT.word = RIGHT.word AND l.wip = RIGHT.wip, HINT(forceRemoteKeyedLookup(true))),
+               IF(forceLocal,
+                  JOIN(searchWords, wordIndex, RIGHT.kind = l.kind AND LEFT.word = RIGHT.word AND l.wip = RIGHT.wip, HINT(remoteKeyedLookup(false))),
+                  JOIN(searchWords, wordIndex, RIGHT.kind = l.kind AND LEFT.word = RIGHT.word AND l.wip = RIGHT.wip)
+                 )
+              );
+      SELF.wPosSum := SUM(kj, wpos);
+      SELF := l;
+    END;
+
+    lhsRecs := 10;
+    lhs := DATASET(lhsRecs, TRANSFORM(rec, SELF.kind := 1; SELF.wip := COUNTER), DISTRIBUTED) + DATASET(lhsRecs, TRANSFORM(rec, SELF.kind := 2; SELF.wip := COUNTER), DISTRIBUTED);
+    RETURN PROJECT(lhs, cqtrans(LEFT));
+  END;
+END;

testing/regress/ecl/key/kjstats-all1.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>9721</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>20</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><name>NumIndexScans</name><value>9721</value></Row>
+ <Row><name>NumIndexSeeks</name><value>1012</value></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><name>NumIndexScans</name><value>32550</value></Row>
+ <Row><name>NumIndexSeeks</name><value>374110</value></Row>
+ <Row><name>NumIndexWildSeeks</name><value>353990</value></Row>
+</Dataset>

testing/regress/ecl/key/kjstats-thor1.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>9721</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>NumIndexScans</name><value>9721</value></Row>
+ <Row><name>NumIndexSeeks</name><value>1012</value></Row>
+</Dataset>

testing/regress/ecl/key/kjstats-thor2.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>20</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>CostFileAccess</name><value>38</value></Row>
+ <Row><name>NumIndexScans</name><value>32550</value></Row>
+ <Row><name>NumIndexSeeks</name><value>374110</value></Row>
+ <Row><name>NumIndexWildSeeks</name><value>353990</value></Row>
+</Dataset>

testing/regress/ecl/kjstats-all1.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie - seeks/wildseeks are ~ double in roxie - investigate
+//nohthor - seeks/wildseeks are slightly different - investigate
+
+//class=file
+//class=index
+
+#onwarning (4522, ignore); // ignore 'Implicit LIMIT(10000) added to keyed join'
+#onwarning (4523, ignore); // ignore 'Neither LIMIT() nor CHOOSEN() supplied for index read on'
+#onwarning (4515, ignore); // ignore 'keyed filter on wip follows unkeyed component wpos in the key'
+
+import ^ as root;
+
+// turn off node fetch timing, in order to get consistent results (inc. costs)
+#option('nodeFetchThresholdNs', 0);
+
+import common.JobStats;
+
+kJStatKinds := DATASET([
+   { 'NumIndexSeeks' }
+  ,{ 'NumIndexScans' }
+  ,{ 'NumIndexWildSeeks' }
+], JobStats.jobStatKindRec);
+
+kJFilter1 := 'scope[w2:graph2],nested[all],where[kind=159]';
+kJFilter2 := 'scope[w2:graph3],nested[all],where[kind=159]';
+
+SEQUENTIAL(
+  COUNT(NOFOLD(JobStats.TestKJSimple()));
+  COUNT(NOFOLD(JobStats.TestKJCQ()));
+  OUTPUT(JobStats.getStats(WORKUNIT, kJFilter1, kJStatKinds), ALL);
+  OUTPUT(JobStats.getStats(WORKUNIT, kJFilter2, kJStatKinds), ALL);
+);

testing/regress/ecl/kjstats-thor1.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie
+//nohthor
+
+//class=file
+//class=index
+//version forceKJLocal=false,forceKJRemote=false
+//version forceKJLocal=false,forceKJRemote=true
+//version forceKJLocal=true,forceKJRemote=false
+
+#onwarning (4522, ignore); // ignore 'Implicit LIMIT(10000) added to keyed join'
+#onwarning (4523, ignore); // ignore 'Neither LIMIT() nor CHOOSEN() supplied for index read on'
+
+import ^ as root;
+forceKJRemote := #IFDEFINED(root.forceKJRemote, false);
+forceKJLocal := #IFDEFINED(root.forceKJLocal, false);
+
+// turn off node fetch timing, in order to get consistent results (inc. costs)
+#option('nodeFetchThresholdNs', 0);
+
+import common.JobStats;
+
+kJStatKinds := DATASET([
+   { 'NumIndexSeeks' }
+  ,{ 'NumIndexScans' }
+  ,{ 'NumIndexWildSeeks' }
+//,{ 'CostFileAccess' } // ignore for now, variance between local and remote handlers - investigate.
+//,{ 'NumLeafCacheAdds' } // ignore for now, remote handler in Thor does not produce this stat.
+], JobStats.jobStatKindRec);
+
+kJFilter := 'scope[w2:graph2],nested[all],where[kind=159]';
+
+SEQUENTIAL(
+  COUNT(NOFOLD(JobStats.TestKJSimple(forceKJRemote, forceKJLocal)));
+  OUTPUT(JobStats.getStats(WORKUNIT, kJFilter, kJStatKinds), ALL);
+);

testing/regress/ecl/kjstats-thor2.ecl

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie
+//nohthor
+
+//class=file
+//class=index
+
+// disabled version because don't match, need revisiting
+//disabled-version forceKJLocal=false,forceKJRemote=false
+//disabled-version forceKJLocal=false,forceKJRemote=true
+//version forceKJLocal=true,forceKJRemote=false
+
+#onwarning (4522, ignore); // ignore 'Implicit LIMIT(10000) added to keyed join'
+#onwarning (4523, ignore); // ignore 'Neither LIMIT() nor CHOOSEN() supplied for index read on'
+#onwarning (4515, ignore); // ignore 'keyed filter on wip follows unkeyed component wpos in the key'
+
+import ^ as root;
+forceKJRemote := #IFDEFINED(root.forceKJRemote, false);
+forceKJLocal := #IFDEFINED(root.forceKJLocal, false);
+
+// turn off node fetch timing, in order to get consistent results (inc. costs)
+#option('nodeFetchThresholdNs', 0);
+
+import common.JobStats;
+
+kJStatKinds := DATASET([
+   { 'NumIndexSeeks' }
+  ,{ 'NumIndexScans' }
+  ,{ 'NumIndexWildSeeks' }
+  ,{ 'CostFileAccess' }
+//,{ 'NumLeafCacheAdds' } // ignore for now, remote handler in Thor does not produce this stat.
+], JobStats.jobStatKindRec);
+
+kJFilter := 'scope[w2:graph2],nested[all],where[kind=159]';
+
+SEQUENTIAL(
+  COUNT(NOFOLD(JobStats.TestKJCQ(forceKJRemote, forceKJLocal)));
+  OUTPUT(JobStats.getStats(WORKUNIT, kJFilter, kJStatKinds), ALL);
+);

thorlcr/activities/keyedjoin/thkeyedjoin.cpp

@@ -310,7 +310,7 @@ class CKeyedJoinMaster : public CMasterActivity
         totalIndexParts = 0;
 
         Owned<IDistributedFile> dataFile;
-        Owned<IDistributedFile> indexFile = lookupReadFile(indexFileName, AccessMode::readRandom, false, false, 0 != (helper->getJoinFlags() & JFindexoptional), true, indexReadActivityStatistics, &indexFileStatsTableEntry);
+        Owned<IDistributedFile> indexFile = lookupReadFile(indexFileName, AccessMode::readRandom, false, false, 0 != (helper->getJoinFlags() & JFindexoptional), true, indexReadFileStatistics, &indexFileStatsTableEntry);
         if (indexFile)
         {
             if (!isFileKey(indexFile))