From 6a17ea10fd4a191c8a7e3a39ef2be2860a538fad Mon Sep 17 00:00:00 2001 From: M Kelly Date: Thu, 14 Nov 2024 09:35:06 -0500 Subject: [PATCH] HPCC-32964 Add a Roxie Background priority queue 2 Signed-off-by: M Kelly --- roxie/ccd/ccd.hpp | 4 ++-- roxie/ccd/ccdmain.cpp | 6 ++---- roxie/ccd/ccdquery.cpp | 5 +++-- roxie/ccd/ccdqueue.cpp | 2 +- roxie/ccd/ccdserver.cpp | 2 +- roxie/ccd/ccdstate.cpp | 10 ---------- 6 files changed, 9 insertions(+), 20 deletions(-) diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index b818edc53d9..b2bce627abf 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -333,8 +333,8 @@ extern unsigned memoryStatsInterval; extern unsigned pingInterval; extern unsigned socketCheckInterval; extern memsize_t defaultMemoryLimit; -extern unsigned defaultTimeLimit[4]; -extern unsigned defaultWarnTimeLimit[4]; +extern unsigned defaultTimeLimit[3]; +extern unsigned defaultWarnTimeLimit[3]; extern unsigned defaultThorConnectTimeout; extern bool pretendAllOpt; extern ClientCertificate clientCert; diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 3d4262b6922..192e58a5670 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -164,8 +164,8 @@ int backgroundCopyPrio = 0; unsigned memoryStatsInterval = 0; memsize_t defaultMemoryLimit; -unsigned defaultTimeLimit[4] = {0, 0, 0, 0}; -unsigned defaultWarnTimeLimit[4] = {0, 5000, 5000, 10000}; +unsigned defaultTimeLimit[3] = {0, 0, 0}; +unsigned defaultWarnTimeLimit[3] = {0, 5000, 5000}; unsigned defaultThorConnectTimeout; unsigned defaultParallelJoinPreload = 0; @@ -1169,11 +1169,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) defaultTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeLimit", 0); defaultTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeLimit", 0); defaultTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeLimit", 0); - defaultTimeLimit[3] = (unsigned) topology->getPropInt64("@defaultBGPriorityTimeLimit", 0); defaultWarnTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeWarning", 0); defaultWarnTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeWarning", 0); defaultWarnTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeWarning", 0); - defaultWarnTimeLimit[3] = (unsigned) topology->getPropInt64("@defaultBGPriorityTimeWarning", 0); defaultThorConnectTimeout = (unsigned) topology->getPropInt64("@defaultThorConnectTimeout", 60); continuationCompressThreshold = (unsigned) topology->getPropInt64("@continuationCompressThreshold", 1024); diff --git a/roxie/ccd/ccdquery.cpp b/roxie/ccd/ccdquery.cpp index 2ec6239e245..67631c107fa 100644 --- a/roxie/ccd/ccdquery.cpp +++ b/roxie/ccd/ccdquery.cpp @@ -402,8 +402,9 @@ void QueryOptions::setFromWorkUnit(IConstWorkUnit &wu, const IPropertyTree *stat updateFromContext(priority, stateInfo, "@priority"); if ((int)priority < 0) { - timeLimit = defaultTimeLimit[3]; - warnTimeLimit = defaultWarnTimeLimit[3]; + // use LOW queue time limits ... + timeLimit = defaultTimeLimit[0]; + warnTimeLimit = defaultWarnTimeLimit[0]; } else { diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index a79264cf54d..9023203d138 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -1936,7 +1936,7 @@ class RoxieReceiverBase : implements IRoxieOutputQueueManager, public CInterface loQueue.start(); hiQueue.start(); slaQueue.start(); - bgQueue.start(); + bgQueue.start(); // consider nice(+3) BG threads } virtual void stop() diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index b4e3cd3f051..e1b3fcb1509 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -4559,7 +4559,7 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie // low priority output Q to not get service on time. unsigned pmask = p->queryHeader().activityId & ROXIE_PRIORITY_MASK; if ((colocalArg == 0) && // not a child query activity?? - (pmask && (pmask != ROXIE_PRIORITY_MASK)) && + (pmask && (pmask != (ROXIE_SLA_PRIORITY + ROXIE_HIGH_PRIORITY))) && (p->queryHeader().overflowSequence == 0) && (p->queryHeader().continueSequence & ~CONTINUE_SEQUENCE_SKIPTO)==0) p->queryHeader().retries |= ROXIE_FASTLANE; diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp index 2108347a486..1b239971d9c 100644 --- a/roxie/ccd/ccdstate.cpp +++ b/roxie/ccd/ccdstate.cpp @@ -2439,16 +2439,6 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme defaultWarnTimeLimit[2] = control->getPropInt("@limit", 0); topology->setPropInt("@defaultSLAPriorityTimeWarning", defaultWarnTimeLimit[2]); } - else if (stricmp(queryName, "control:defaultBGPriorityTimeLimit")==0) - { - defaultTimeLimit[3] = control->getPropInt("@limit", 0); - topology->setPropInt("@defaultBGPriorityTimeLimit", defaultTimeLimit[3]); - } - else if (stricmp(queryName, "control:defaultBGPriorityTimeWarning")==0) - { - defaultWarnTimeLimit[3] = control->getPropInt("@limit", 0); - topology->setPropInt("@defaultBGPriorityTimeWarning", defaultWarnTimeLimit[3]); - } else if (stricmp(queryName, "control:deleteUnneededPhysicalFiles")==0) { UNIMPLEMENTED;