diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml
index a29f49f26e1..aa3cf3255b2 100644
--- a/.github/workflows/build-vcpkg.yml
+++ b/.github/workflows/build-vcpkg.yml
@@ -123,6 +123,36 @@ jobs:
asset-name: 'docker-ubuntu-22_04'
generate-zap: ""
secrets: inherit
+
+ check-documentation-changes:
+ if: ${{ contains('pull_request,push', github.event_name) }}
+ runs-on: ubuntu-22.04
+ outputs:
+ documentation_contents_changed: ${{ steps.variables.outputs.documentation_contents_changed }}
+ steps:
+ - name: Check for Documentation Changes
+ id: changed
+ uses: dorny/paths-filter@v3
+ with:
+ filters: |
+ src:
+ - 'docs/**'
+ - '.github/workflows/test-documentation.yml'
+ - name: Set Output
+ id: variables
+ run: |
+ echo "documentation_contents_changed=${{ steps.changed.outputs.src }}" >> $GITHUB_OUTPUT
+ - name: Print Variables
+ run: |
+ echo "${{ toJSON(steps.variables.outputs)}}"
+
+ test-documentation-ubuntu-22_04:
+ needs: check-documentation-changes
+ if: ${{ contains('pull_request,push', github.event_name) && needs.check-documentation-changes.outputs.documentation_contents_changed == 'true' }}
+ uses: ./.github/workflows/test-documentation.yml
+ with:
+ os: 'ubuntu-22.04'
+ asset-name: 'Documentation'
build-docker-ubuntu-20_04:
if: ${{ contains('schedule,push', github.event_name) }}
diff --git a/.github/workflows/test-documentation.yml b/.github/workflows/test-documentation.yml
new file mode 100644
index 00000000000..db2b8831757
--- /dev/null
+++ b/.github/workflows/test-documentation.yml
@@ -0,0 +1,108 @@
+name: Build Documentation
+
+on:
+ workflow_call:
+ inputs:
+ os:
+ type: string
+ description: 'Operating System'
+ required: false
+ default: 'ubuntu-22.04'
+ asset-name:
+ type: string
+ description: 'Asset Name'
+ required: false
+ default: 'Documentation'
+
+ workflow_dispatch:
+ inputs:
+ os:
+ type: string
+ description: 'Operating System'
+ required: false
+ default: 'ubuntu-22.04'
+ asset-name:
+ type: string
+ description: 'Asset Name'
+ required: false
+ default: 'Documentation'
+
+jobs:
+ build-documentation:
+ name: Build Documentation
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Checkout HPCC-Platform
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.ref }}
+ submodules: recursive
+ path: ${{ github.workspace }}/HPCC-Platform
+
+ - name: Calculate vars
+ id: vars
+ working-directory: ${{ github.workspace }}/HPCC-Platform/vcpkg
+ run: |
+ vcpkg_sha_short=$(git rev-parse --short=8 HEAD)
+ echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT
+ docker_build_label=hpccsystems/platform-build-${{ inputs.os }}
+ echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT
+ echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT
+ community_base_ref=${{ github.event.base_ref || github.ref }}
+ candidate_branch=$(echo $community_base_ref | cut -d'/' -f3)
+ candidate_base_branch=$(echo $candidate_branch | awk -F'.' -v OFS='.' '{ $3="x"; print }')
+ echo "docker_tag_candidate_base=$docker_build_label:$candidate_base_branch" >> $GITHUB_OUTPUT
+ community_ref=${{ github.ref }}
+ community_tag=$(echo $community_ref | cut -d'/' -f3)
+ echo "community_tag=$community_tag" >> $GITHUB_OUTPUT
+
+
+ - name: Print vars
+ run: |
+ echo "${{ toJSON(steps.vars.outputs) }})"
+
+
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Create Build Image
+ uses: docker/build-push-action@v5
+ with:
+ builder: ${{ steps.buildx.outputs.name }}
+ file: ${{ github.workspace }}/HPCC-Platform/dockerfiles/vcpkg/${{ inputs.os }}.dockerfile
+ context: ${{ github.workspace }}/HPCC-Platform/dockerfiles/vcpkg
+ push: false
+ load: true
+ build-args: |
+ VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}
+ tags: |
+ ${{ steps.vars.outputs.docker_tag_candidate_base }}
+ cache-from: |
+ type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }}
+ type=registry,ref=${{ steps.vars.outputs.docker_tag }}
+ cache-to: type=inline
+
+ - name: CMake documentation
+ run: |
+ mkdir -p {${{ github.workspace }}/build,EN_US,PT_BR}
+ docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
+ cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DVCPKG_FILES_DIR=/hpcc-dev -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
+ cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
+ docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-$(echo '${{ steps.vars.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
+ docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-$(echo '${{ steps.vars.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
+
+ - name: Upload build artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ inputs.asset-name }}
+ path: |
+ ${{ github.workspace }}/build/Release/docs/*.zip
+ ${{ github.workspace }}/build/Release/docs/EN_US/*.zip
+ ${{ github.workspace }}/build/Release/docs/PT_BR/*.zip
+ ${{ github.workspace }}/build/docs/EN_US/EclipseHelp/*.zip
+ ${{ github.workspace }}/build/docs/EN_US/HTMLHelp/*.zip
+ ${{ github.workspace }}/build/docs/PT_BR/HTMLHelp/*.zip
+ compression-level: 0
+
diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
index a86fbf6a097..c7e3df41b05 100644
--- a/common/thorhelper/thorsoapcall.cpp
+++ b/common/thorhelper/thorsoapcall.cpp
@@ -2466,9 +2466,15 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
checkTimeLimitExceeded(&remainingMS);
Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0);
+ CCycleTimer dnsTimer;
+
// TODO: for DNS, do we use timeoutMS or remainingMS or remainingMS / maxRetries+1 or ?
ep.set(connUrl.host.get(), connUrl.port, master->timeoutMS);
+ unsigned __int64 dnsNs = dnsTimer.elapsedNs();
+ master->logctx.noteStatistic(StTimeSoapcallDNS, dnsNs);
+ master->activitySpanScope->setSpanAttribute("SoapcallDNSTimeNs", dnsNs);
+
if (ep.isNull())
throw MakeStringException(-1, "Failed to resolve host '%s'", nullText(connUrl.host.get()));
@@ -2489,6 +2495,8 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
isReused = false;
keepAlive = true;
+ CCycleTimer connTimer;
+
// TODO: for each connect attempt, do we use timeoutMS or remainingMS or remainingMS / maxRetries or ?
socket.setown(blacklist->connect(ep, master->logctx, (unsigned)master->maxRetries, master->timeoutMS, master->roxieAbortMonitor, master->rowProvider));
@@ -2518,11 +2526,17 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
throw makeStringException(0, err.str());
#endif
}
+
+ unsigned __int64 connNs = connTimer.elapsedNs();
+ master->logctx.noteStatistic(StTimeSoapcallConnect, connNs);
+ master->activitySpanScope->setSpanAttribute("SoapcallConnectTimeNs", connNs);
}
break;
}
catch (IException *e)
{
+ master->logctx.noteStatistic(StNumSoapcallConnectFailures, 1);
+
if (master->timeLimitExceeded)
{
master->activitySpanScope->recordError(SpanError("Time Limit Exceeded", e->errorCode(), true, true));
diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV1Blob.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV1Blob.yaml
new file mode 100644
index 00000000000..0750e38e896
--- /dev/null
+++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV1Blob.yaml
@@ -0,0 +1,103 @@
+# Configures HPCC logAccess to target Azure Log Analytics Workspace
+global:
+ logAccess:
+ name: "Azure LogAnalytics LogAccess"
+ type: "AzureLogAnalyticsCurl"
+ blobMode: "true"
+ #connection:
+ #All connection attributes are optional.
+ #It is preferable to provide connection values as secret values category 'esp', secret name 'azure_logaccess'
+ # NOTE: secret 'azure_logaccess' must include 'aad-client-secret' and it cannot be provided in configuration
+ #
+ #workspaceID: "XYZ" #ID of the Azure LogAnalytics workspace to query logs from
+ # Secret value equivalent: 'ala-workspace-id'
+ #clientID: "DEF" #ID of Azure Active Directory registered application with api.loganalytics.io access - format: 00000000-0000-0000-0000-000000000000
+ # Secret value equivalent: 'aad-client-id'
+ #tenantID: "ABC" #The Azure Active Directory Tenant ID, required for KQL API access
+ # Secret value equivalent: 'aad-tenant-id'
+ logMaps:
+ - type: "global"
+ storeName: "ContainerLog"
+ searchColumn: "LogEntry"
+ columnType: "string"
+ columnMode: "MIN"
+ timeStampColumn: "TimeGenerated"
+ - type: "workunits"
+ searchColumn: "LogEntry"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "components"
+ storeName: "ContainerInventory"
+ searchColumn: "Name"
+ keyColumn: "ContainerID"
+ columnMode: "MIN"
+ columnType: "string"
+ timeStampColumn: "TimeGenerated"
+ disableJoins: false #Potentially expensive join operations needed to fetch a certain column can be disabled
+ - type: "audience"
+ searchColumn: "LogEntry"
+ enumValues:
+ - code: OPR
+ - code: USR
+ - code: PRO
+ - code: ADT
+ - code: MON
+ columnMode: "DEFAULT"
+ columnType: "enum"
+ - type: "class"
+ searchColumn: "LogEntry"
+ enumValues:
+ - code: DIS
+ - code: ERR
+ - code: WRN
+ - code: INF
+ - code: PRO
+ - code: MET
+ - code: EVT
+ columnMode: "DEFAULT"
+ columnType: "enum"
+ - type: "instance"
+ columnMode: "DEFAULT"
+ searchColumn: "Computer"
+ columnMode: "ALL"
+ columnType: "string"
+ - type: "message"
+ searchColumn: "LogEntry"
+ columnMode: "MIN"
+ columnType: "string"
+ - type: "logid"
+ searchColumn: "LogEntry"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "processid"
+ searchColumn: "LogEntry"
+ columnMode: "ALL"
+ columnType: "string"
+ - type: "threadid"
+ searchColumn: "LogEntry"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "timestamp"
+ searchColumn: "TimeGenerated"
+ columnMode: "MIN"
+ columnType: "Timestamp"
+ #- type: "pod"
+ # searchColumn: "PodName"
+ # columnMode: "DEFAULT"
+ # columnType: "string"
+ - type: "spanid"
+ searchColumn: "LogEntry"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "traceid"
+ searchColumn: "LogEntry"
+ columnMode: "DEFAULT"
+ columnType: "string"
+secrets:
+ esp:
+ azure-logaccess: "azure-logaccess"
+vaults:
+ esp:
+ - name: my-azure-logaccess-vault
+ url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+ kind: kv-v2
diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2Blob.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2Blob.yaml
new file mode 100644
index 00000000000..89b73d19b05
--- /dev/null
+++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2Blob.yaml
@@ -0,0 +1,101 @@
+# Configures HPCC logAccess to target Azure Log Analytics Workspace
+global:
+ logAccess:
+ name: "Azure LogAnalytics LogAccess"
+ type: "AzureLogAnalyticsCurl"
+ blobMode: "true"
+ #connection:
+ #All connection attributes are optional.
+ #It is preferable to provide connection values as secret values category 'esp', secret name 'azure_logaccess'
+ # NOTE: secret 'azure_logaccess' must include 'aad-client-secret' and it cannot be provided in configuration
+ #
+ #workspaceID: "XYZ" #ID of the Azure LogAnalytics workspace to query logs from
+ # Secret value equivalent: 'ala-workspace-id'
+ #clientID: "DEF" #ID of Azure Active Directory registered application with api.loganalytics.io access - format: 00000000-0000-0000-0000-000000000000
+ # Secret value equivalent: 'aad-client-id'
+ #tenantID: "ABC" #The Azure Active Directory Tenant ID, required for KQL API access
+ # Secret value equivalent: 'aad-tenant-id'
+ logMaps:
+ - type: "global"
+ storeName: "ContainerLogV2"
+ searchColumn: "LogMessage"
+ columnType: "dynamic"
+ columnMode: "MIN"
+ timeStampColumn: "TimeGenerated"
+ - type: "workunits"
+ searchColumn: "LogMessage"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "components"
+ storeName: "ContainerLogV2"
+ searchColumn: "ContainerName" # Container name happens to coincide with component name
+ keyColumn: "ContainerName"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "audience"
+ searchColumn: "LogMessage"
+ enumValues:
+ - code: OPR
+ - code: USR
+ - code: PRO
+ - code: ADT
+ - code: MON
+ columnMode: "DEFAULT"
+ columnType: "enum"
+ - type: "class"
+ searchColumn: "LogMessage"
+ enumValues:
+ - code: DIS
+ - code: ERR
+ - code: WRN
+ - code: INF
+ - code: PRO
+ - code: MET
+ - code: EVT
+ columnMode: "DEFAULT"
+ columnType: "enum"
+ - type: "instance"
+ columnMode: "DEFAULT"
+ searchColumn: "Computer"
+ columnMode: "ALL"
+ columnType: "string"
+ - type: "message"
+ searchColumn: "LogMessage"
+ columnMode: "MIN"
+ columnType: "string"
+ - type: "logid"
+ searchColumn: "LogMessage"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "processid"
+ searchColumn: "LogMessage"
+ columnMode: "ALL"
+ columnType: "string"
+ - type: "threadid"
+ searchColumn: "LogMessage"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "timestamp"
+ searchColumn: "LogMessage"
+ columnMode: "MIN"
+ columnType: "string"
+ - type: "pod"
+ searchColumn: "PodName"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "spanid"
+ searchColumn: "LogMessage"
+ columnMode: "DEFAULT"
+ columnType: "string"
+ - type: "traceid"
+ searchColumn: "LogMessage"
+ columnMode: "DEFAULT"
+ columnType: "string"
+secrets:
+ esp:
+ azure-logaccess: "azure-logaccess"
+vaults:
+ esp:
+ - name: my-azure-logaccess-vault
+ url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+ kind: kv-v2
diff --git a/helm/managed/logging/loki-stack/README.md b/helm/managed/logging/loki-stack/README.md
index 8d2af452a8d..e77f524bd53 100644
--- a/helm/managed/logging/loki-stack/README.md
+++ b/helm/managed/logging/loki-stack/README.md
@@ -10,7 +10,8 @@ A Loki Datasource is created automatically, which allowers users to monitor/quer
### Helm Deployment
To deploy the light-weight Loki Stack for HPCC component log processing issue the following command:
->helm install myloki HPCC-Systems/helm/managed/logging/loki-stack/
+>helm install myloki4hpcclogs HPCC-Systems/helm/managed/logging/loki-stack/
+Note: the deployment name 'myloki4hpcclogs' is customizable; however, any changes need to be reflected in the LogAccess configuration (See section on configuring LogAccess below)
### Dependencies
This chart is dependent on the Grafana Loki-stack Helm charts which in turn is dependent on Loki, Grafana, Promtail.
@@ -23,7 +24,9 @@ Helm provides a convenient command to automatically pull appropriate dependencie
##### HELM Install parameter
Otherwise, provide the "--dependency-update" argument in the helm install command
For example:
-> helm install myloki HPCC-Systems/helm/managed/logging/loki-stack/ --dependency-update
+> helm install myloki4hpcclogs HPCC-Systems/helm/managed/logging/loki-stack/ --dependency-update
+
+Note: the deployment name 'myloki4hpcclogs' is customizable; however, any changes need to be reflected in the LogAccess configuration (See section on configuring LogAccess below)
### Components
Grafana Loki Stack is comprised of a set of components that which serve as a full-featured logging stack.
@@ -172,7 +175,7 @@ username: 5 bytes
The target HPCC deployment should be directed to use the desired Grafana endpoint with the Loki datasource, and the newly created secret by providing appropriate logAccess values (such as ./grafana-hpcc-logaccess.yaml).
-Example use:
+Example use for targeting a loki stack deployed as 'myloki4hpcclogs' on the default namespace:
```
helm install myhpcc hpcc/hpcc -f HPCC-Platform/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml
@@ -182,8 +185,10 @@ Example use:
The grafana hpcc logaccess values should provide Grafana connection information, such as the host, and port; the Loki datasource where the logs reside; the k8s namespace under which the logs were created (non-default namespace highly recommended); and the hpcc component log format (table|json|xml)
+Example values file describing logAccess targeting loki stack deployed as 'myloki4hpcclogs' on the default namespace. Note that the "host" entry must reflect the name of the deployed Loki stack, as shown in the excerpt below (eg **_myloki4hpcclogs_**-grafana.default.svc.cluster.local):
+
```
-Example use:
+
global:
logAccess:
name: "Grafana/loki stack log access"
@@ -220,4 +225,4 @@ For example:
- ```
\ No newline at end of file
+ ```
diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp
index 471dc802cb5..5d9b47ea209 100644
--- a/roxie/ccd/ccdqueue.cpp
+++ b/roxie/ccd/ccdqueue.cpp
@@ -2434,6 +2434,31 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
DelayedPacketQueueManager delayed;
#endif
+ class WorkerUdpTracker : public TimeDivisionTracker<6, false>
+ {
+ public:
+ enum
+ {
+ other,
+ waiting,
+ allocating,
+ processing,
+ pushing,
+ checkingRunning
+ };
+
+ WorkerUdpTracker(const char *name, unsigned reportIntervalSeconds) : TimeDivisionTracker<6, false>(name, reportIntervalSeconds)
+ {
+ stateNames[other] = "other";
+ stateNames[waiting] = "waiting";
+ stateNames[allocating] = "allocating";
+ stateNames[processing] = "processing";
+ stateNames[pushing] = "pushing";
+ stateNames[checkingRunning] = "checking running";
+ }
+
+ } timeTracker;
+
class ReceiverThread : public Thread
{
RoxieSocketQueueManager &parent;
@@ -2453,7 +2478,7 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
} readThread;
public:
- RoxieSocketQueueManager(unsigned _numWorkers) : RoxieReceiverBase(_numWorkers), logctx("RoxieSocketQueueManager"), readThread(*this)
+ RoxieSocketQueueManager(unsigned _numWorkers) : RoxieReceiverBase(_numWorkers), logctx("RoxieSocketQueueManager"), timeTracker("WorkerUdpReader", 60), readThread(*this)
{
maxPacketSize = multicastSocket->get_max_send_size();
if ((maxPacketSize==0)||(maxPacketSize>65535))
@@ -2765,21 +2790,26 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
// if found, send an IBYTI and discard retry request
bool alreadyRunning = false;
- Owned wi = queue.running();
- ForEach(*wi)
{
- CRoxieWorker &w = (CRoxieWorker &) wi->query();
- if (w.match(header))
+ WorkerUdpTracker::TimeDivision division(timeTracker, WorkerUdpTracker::checkingRunning);
+
+ Owned wi = queue.running();
+ ForEach(*wi)
{
- alreadyRunning = true;
- ROQ->sendIbyti(header, logctx, mySubchannel);
- if (doTrace(traceRoxiePackets, TraceFlags::Max))
+ CRoxieWorker &w = (CRoxieWorker &) wi->query();
+ if (w.match(header))
{
- StringBuffer xx; logctx.CTXLOG("Ignored retry on subchannel %u for running activity %s", mySubchannel, header.toString(xx).str());
+ alreadyRunning = true;
+ ROQ->sendIbyti(header, logctx, mySubchannel);
+ if (doTrace(traceRoxiePackets, TraceFlags::Max))
+ {
+ StringBuffer xx; logctx.CTXLOG("Ignored retry on subchannel %u for running activity %s", mySubchannel, header.toString(xx).str());
+ }
+ break;
}
- break;
}
}
+
if (!alreadyRunning && checkCompleted && ROQ->replyPending(header))
{
alreadyRunning = true;
@@ -2795,6 +2825,7 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
{
StringBuffer xx; logctx.CTXLOG("Retry %d received on subchannel %u for %s", retries+1, mySubchannel, header.toString(xx).str());
}
+ WorkerUdpTracker::TimeDivision division(timeTracker, WorkerUdpTracker::pushing);
#ifdef NEW_IBYTI
// It's debatable whether we should delay for the primary here - they had one chance already...
// But then again, so did we, assuming the timeout is longer than the IBYTIdelay
@@ -2813,6 +2844,7 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
}
else // first time (not a retry).
{
+ WorkerUdpTracker::TimeDivision division(timeTracker, WorkerUdpTracker::pushing);
#ifdef NEW_IBYTI
unsigned delay = 0;
if (mySubchannel != 0 && (header.activityId & ~ROXIE_PRIORITY_MASK) < ROXIE_ACTIVITY_SPECIAL_FIRST) // i.e. I am not the primary here, and never delay special
@@ -2837,6 +2869,7 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
doIbytiDelay?"YES":"NO", minIbytiDelay, initIbytiDelay);
MemoryBuffer mb;
+ WorkerUdpTracker::TimeDivision division(timeTracker, WorkerUdpTracker::other);
for (;;)
{
mb.clear();
@@ -2851,8 +2884,14 @@ class RoxieSocketQueueManager : public RoxieReceiverBase
#else
unsigned timeout = 5000;
#endif
+ division.switchState(WorkerUdpTracker::allocating);
+ void * buffer = mb.reserve(maxPacketSize);
+
+ division.switchState(WorkerUdpTracker::waiting);
unsigned l;
- multicastSocket->readtms(mb.reserve(maxPacketSize), sizeof(RoxiePacketHeader), maxPacketSize, l, timeout);
+ multicastSocket->readtms(buffer, sizeof(RoxiePacketHeader), maxPacketSize, l, timeout);
+ division.switchState(WorkerUdpTracker::processing);
+
mb.setLength(l);
RoxiePacketHeader &header = *(RoxiePacketHeader *) mb.toByteArray();
if (l != header.packetlength)
diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp
index c9aff42310a..60aa3ae2c0f 100644
--- a/roxie/ccd/ccdserver.cpp
+++ b/roxie/ccd/ccdserver.cpp
@@ -496,7 +496,7 @@ static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndex
static const StatisticsMapping diskStatistics({StNumServerCacheHits, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries, StNumAgentRequests, StSizeAgentRequests,
StSizeContinuationData, StNumContinuationRequests }, actStatistics);
-static const StatisticsMapping soapStatistics({ StTimeSoapcall }, actStatistics);
+static const StatisticsMapping soapStatistics({ StTimeSoapcall, StTimeSoapcallDNS, StTimeSoapcallConnect, StNumSoapcallConnectFailures }, actStatistics);
static const StatisticsMapping groupStatistics({ StNumGroups, StNumGroupMax }, actStatistics);
static const StatisticsMapping sortStatistics({ StTimeSortElapsed }, actStatistics);
static const StatisticsMapping indexWriteStatistics({ StNumDuplicateKeys, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds }, actStatistics);
@@ -518,7 +518,7 @@ extern const StatisticsMapping accumulatedStatistics({StWhenFirstRow, StTimeLoca
StCycleBlobFetchCycles, StCycleLeafFetchCycles, StCycleNodeFetchCycles, StTimeBlobFetch, StTimeLeafFetch, StTimeNodeFetch,
StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches,
StNumDiskRejected, StSizeAgentReply, StTimeAgentWait,
- StTimeSoapcall,
+ StTimeSoapcall, StTimeSoapcallDNS, StTimeSoapcallConnect, StNumSoapcallConnectFailures,
StNumGroups,
StTimeSortElapsed,
StNumDuplicateKeys,
diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h
index 1b40a879fe8..bfabe4648cf 100644
--- a/system/jlib/jstatcodes.h
+++ b/system/jlib/jstatcodes.h
@@ -314,6 +314,11 @@ enum StatisticKind
StNumParallelExecute,
StNumAgentRequests,
StSizeAgentRequests,
+ StTimeSoapcallDNS, // Time spent in DNS lookups for soapcalls
+ StTimeSoapcallConnect, // Time spent in connect[+SSL_connect] for soapcalls
+ StCycleSoapcallDNSCycles,
+ StCycleSoapcallConnectCycles,
+ StNumSoapcallConnectFailures,
StMax,
//For any quantity there is potentially the following variants.
diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp
index a29dbee7a76..cb927bdae69 100644
--- a/system/jlib/jstats.cpp
+++ b/system/jlib/jstats.cpp
@@ -986,6 +986,11 @@ static const constexpr StatisticMeta statsMetaData[StMax] = {
{ NUMSTAT(ParallelExecute), "The number of parallel execution paths for this activity" },
{ NUMSTAT(AgentRequests), "The number of agent request packets for this activity" },
{ SIZESTAT(AgentRequests), "The total size of agent request packets for this activity" },
+ { TIMESTAT(SoapcallDNS), "The time taken for DNS lookup in SOAPCALL" },
+ { TIMESTAT(SoapcallConnect), "The time taken for connect[+SSL_connect] in SOAPCALL" },
+ { CYCLESTAT(SoapcallDNS) },
+ { CYCLESTAT(SoapcallConnect) },
+ { NUMSTAT(SoapcallConnectFailures), "The number of SOAPCALL connect failures" },
};
static MapStringTo statisticNameMap(true);
diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
index 9269737a836..35b1decae34 100644
--- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
+++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
@@ -336,6 +336,10 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce
m_pluginCfg.set(&logAccessPluginConfig);
+ //blobMode is a flag to determine if the log entry is to be parsed or not
+ m_blobMode = logAccessPluginConfig.getPropBool("@blobMode", false);
+ DBGLOG("%s: Blob Mode: %s", COMPONENT_NAME, m_blobMode ? "Enabled" : "Disabled");
+
m_globalIndexTimestampField.set(defaultHPCCLogTimeStampCol);
m_globalIndexSearchPattern.set(defaultIndexPattern);
m_globalSearchColName.set(defaultHPCCLogMessageCol);
@@ -483,7 +487,11 @@ void AzureLogAnalyticsCurlClient::getMinReturnColumns(StringBuffer & columns, co
columns.append(defaultHPCCLogComponentCol);
columns.append(", ");
}
- columns.appendf("%s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol);
+
+ if (m_blobMode)
+ columns.appendf("%s, %s", m_globalIndexTimestampField.str(), m_globalSearchColName.str());
+ else
+ columns.appendf("%s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol);
}
void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns, const bool includeComponentName)
@@ -517,12 +525,15 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns
if (!isEmptyString(m_podSearchColName))
columns.appendf("%s, ", m_podSearchColName.str());
- columns.appendf("%s, %s, %s, %s, %s, %s, %s, %s, %s, %s",
- m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(),
- m_audienceSearchColName.str(), m_workunitSearchColName.str(), m_traceSearchColName.str(), m_spanSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol, defaultHPCCLogProcIDCol);
+ if (m_blobMode)
+ columns.appendf("%s, %s", m_globalIndexTimestampField.str(), m_globalSearchColName.str());
+ else
+ columns.appendf("%s, %s, %s, %s, %s, %s, %s, %s, %s, %s",
+ m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(),
+ m_audienceSearchColName.str(), m_workunitSearchColName.str(), m_traceSearchColName.str(), m_spanSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol, defaultHPCCLogProcIDCol);
}
-bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2)
+bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2, bool blobMode)
{
if (isEmptyString(colName))
{
@@ -538,25 +549,32 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName,
else
throw makeStringExceptionV(-1, "%s: Invalid Azure Log Analytics log message column name detected: '%s'. Review logAccess configuration.", COMPONENT_NAME, colName);
- kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s*([0-9a-fA-F]{32}|UNK)?\\s*([0-9a-fA-F]{16}|UNK)?\\s+)?\\\"(.*)\\\"$', %s)[0]", sourceCol.str());
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[0])", defaultHPCCLogSeqCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[1])", defaultHPCCLogAudCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[2])", defaultHPCCLogTypeCol);
- kql.appendf("\n| extend %s = todatetime(hpcclogfields.[3])", defaultHPCCLogTimeStampCol);
- kql.appendf("\n| extend %s = toint(hpcclogfields.[4])", defaultHPCCLogProcIDCol);
- kql.appendf("\n| extend %s = toint(hpcclogfields.[5])", defaultHPCCLogThreadIDCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[6])", defaultHPCCLogJobIDCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[7])", defaultHPCCLogTraceIDCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[8])", defaultHPCCLogSpanIDCol);
- kql.appendf("\n| extend %s = tostring(hpcclogfields.[9])", defaultHPCCLogMessageCol);
- kql.appendf("\n| project-away hpcclogfields, Type, TenantId, _ResourceId, %s, ", colName);
+ if (!blobMode)
+ {
+ kql.appendf("\n| extend hpcclogfields = extract_all(@\'^([0-9A-Fa-f]+)\\s+(OPR|USR|PRG|AUD|UNK)\\s+(DIS|ERR|WRN|INF|PRO|MET|UNK)\\s+(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(UNK|[A-Z]\\d{8}-\\d{6}(?:-\\d+)?)\\s*([0-9a-fA-F]{32}|UNK)?\\s*([0-9a-fA-F]{16}|UNK)?\\s+)?\\\"(.*)\\\"$', %s)[0]", sourceCol.str());
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[0])", defaultHPCCLogSeqCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[1])", defaultHPCCLogAudCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[2])", defaultHPCCLogTypeCol);
+ kql.appendf("\n| extend %s = todatetime(hpcclogfields.[3])", defaultHPCCLogTimeStampCol);
+ kql.appendf("\n| extend %s = toint(hpcclogfields.[4])", defaultHPCCLogProcIDCol);
+ kql.appendf("\n| extend %s = toint(hpcclogfields.[5])", defaultHPCCLogThreadIDCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[6])", defaultHPCCLogJobIDCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[7])", defaultHPCCLogTraceIDCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[8])", defaultHPCCLogSpanIDCol);
+ kql.appendf("\n| extend %s = tostring(hpcclogfields.[9])", defaultHPCCLogMessageCol);
+
+ kql.appendf("\n| project-away hpcclogfields, Type, TenantId, _ResourceId, %s, ", colName);
+ }
+ else
+ {
+ kql.appendf("\n| project-away Type, TenantId, _ResourceId, ");
+ }
if (targetsV2)
kql.append("LogSource, SourceSystem");
else
kql.append("LogEntrySource, TimeOfCommand, SourceSystem");
-
return true;
}
@@ -659,6 +677,11 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
std::string queryField = m_globalSearchColName.str();
std::string queryOperator = " =~ ";
+ if (m_blobMode)
+ {
+ queryOperator = " has ";
+ }
+
filter->toString(queryValue);
switch (filter->filterType())
{
@@ -879,7 +902,8 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
declareContainerIndexJoinTable(queryString, options);
queryString.append(queryIndex);
- generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str(), targetIsContainerLogV2);
+ //this used to parse m_globalSearchColName into hpcc.log.* fields, now just does a project-away
+ generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str(), targetIsContainerLogV2, m_blobMode);
if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter
{
@@ -895,7 +919,6 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
StringBuffer range;
azureLogAnalyticsTimestampQueryRangeString(range, m_globalIndexTimestampField.str(), trange.getStartt().getSimple(),trange.getEndt().isNull() ? -1 : trange.getEndt().getSimple());
queryString.append("\n| where ").append(range.str());
- //if (includeComponentName)
if (!m_disableComponentNameJoins && !targetIsContainerLogV2)
queryString.append("\n) on ").append(m_componentsLookupKeyColumn);
diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
index d2b92460f4e..013cb0e5032 100644
--- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
+++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
@@ -77,6 +77,7 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf
StringBuffer m_instanceLookupKeyColumn;
bool targetIsContainerLogV2 = false;
+ bool m_blobMode = false;
public:
AzureLogAnalyticsCurlClient(IPropertyTree & logAccessPluginConfig);
diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp
index 282c45c2db6..9c6cda182b9 100644
--- a/thorlcr/thorutil/thormisc.cpp
+++ b/thorlcr/thorutil/thormisc.cpp
@@ -75,7 +75,7 @@ static Owned ClusterMPAllocator;
// stat. mappings shared between master and slave activities
const StatisticsMapping spillStatistics({StTimeSpillElapsed, StTimeSortElapsed, StNumSpills, StSizeSpillFile, StSizePeakTempDisk});
const StatisticsMapping executeStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked});
-const StatisticsMapping soapcallStatistics({StTimeSoapcall});
+const StatisticsMapping soapcallStatistics({StTimeSoapcall, StTimeSoapcallDNS, StTimeSoapcallConnect, StNumSoapcallConnectFailures});
const StatisticsMapping basicActivityStatistics({StNumParallelExecute}, executeStatistics, spillStatistics);
const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics);
const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics);
diff --git a/tools/backupnode/backupnode.cpp b/tools/backupnode/backupnode.cpp
index 3f2406c1991..a0929021741 100644
--- a/tools/backupnode/backupnode.cpp
+++ b/tools/backupnode/backupnode.cpp
@@ -532,6 +532,9 @@ int main(int argc, const char *argv[])
StringAttr errdatdir;
StringArray args;
unsigned slaveNum = 0;
+
+ initNullConfiguration();
+
unsigned argNo = 1;
while ((int)argNo