Adding functionality to add B3 propagator to the OtelWebserver NginxM…

…odule (#403)

* B3 propagator for Otel Webserver Nginx Module

* finalize

* readme update

* improvements

* final domezz

* removing excess functions

* done

.gitignore

@@ -42,4 +42,4 @@
 /out
 /out.*
 # Indicator that the tools were deployed
-.buildtools
+.buildtools

instrumentation/otel-webserver-module/README.md

@@ -179,6 +179,7 @@ Currently, Nginx Webserver module monitores some fixed set of modules, which get
 |*NginxModuleTrustIncomingSpans*           | ON               | OPTIONAL: Specify if you want to correlate Nginx instrumented traces and spans with incoming requests.|
 |*NginxModuleAttributes*           |                | OPTIONAL: Can be used to pass additionalccustom attributes to the span, nginx variables are also supported. All elements must be separated by a space, and different attribute key value pairs must be separated by a comma( even the comma needs to be separated by space). e.g. ```NginxModuleAttributes     Key1 Value1 , Key2 $request_uri;``` |
 |*NginxModuleIgnorePaths*           |               | OPTIONAL: Request URIs matching the Regex will not be monitored. Multiple space separated Regex can be provided( `'\'` symbol needs to be used carefully, Nginx treats `'\'` as a escape sequence, thus if the Regex sequence contains a `'\'`, it need to be replaced by `'\\'`, likewise if sequence contains `'\\'`, it need to be written as `'\\\\'` e.g. `.*\.html` -> `.*\\.html` ) e.g. ```NginxModuleIgnorePaths               .*\\.html /test_.*;```|
+|*NginxModulePropagatorType*                   | w3c                 | OPTIONAL: Specify the Propagator used by the instrumentation (W3C and B3 propagators available). e.g.```NginxModulePropagatorType                  b3;```|
 
 
 

instrumentation/otel-webserver-module/include/core/api/OpentelemetrySdk.h

@@ -33,6 +33,7 @@
 #define OTEL_SDK_ENV_OTEL_PROCESSOR_TYPE "OTEL_SDK_ENV_OTEL_PROCESSOR_TYPE"
 #define OTEL_SDK_ENV_OTEL_SAMPLER_TYPE "OTEL_SDK_ENV_OTEL_SAMPLER_TYPE"
 #define OTEL_SDK_ENV_OTEL_LIBRARY_NAME "OTEL_SDK_ENV_OTEL_LIBRARY_NAME"
+#define OTEL_SDK_ENV_OTEL_PROPAGATOR_TYPE "OTEL_SDK_ENV_OTEL_PROPAGATOR_TYPE"
 /* {{{ For API user: optional, only if connection by aggregator is required */
 #define OTEL_SDK_ENV_SERVICE_NAMESPACE "OTEL_SDK_ENV_SERVICE_NAMESPACE"
 #define OTEL_SDK_ENV_SERVICE_NAME "OTEL_SDK_ENV_SERVICE_NAME"                   /*optional*/

instrumentation/otel-webserver-module/include/core/api/TenantConfig.h

@@ -47,6 +47,7 @@ class TenantConfig
     const std::string& getOtelExporterEndpoint() const {return otelExporterEndpoint;}
     const std::string& getOtelExporterOtlpHeaders() const {return otelExporterOtlpHeaders;}
     const std::string& getOtelProcessorType() const {return otelProcessorType;}
+    const std::string& getOtelPropagatorType() const {return otelPropagatorType;}
     const unsigned getOtelMaxQueueSize() const {return otelMaxQueueSize;}
     const unsigned getOtelScheduledDelayMillis() const {return otelScheduledDelayMillis;}
     const unsigned getOtelExportTimeoutMillis() const {return otelExportTimeoutMillis;}
@@ -64,6 +65,7 @@ class TenantConfig
     void setOtelExporterEndpoint(const std::string& otelExporterEndpoint) { this->otelExporterEndpoint = otelExporterEndpoint; }
     void setOtelExporterOtlpHeaders(const std::string& otelExporterOtlpHeaders) { this->otelExporterOtlpHeaders = otelExporterOtlpHeaders; }
     void setOtelProcessorType(const std::string& otelProcessorType) { this->otelProcessorType = otelProcessorType; }
+    void setOtelPropagatorType(const std::string& otelPropagatorType) { this->otelPropagatorType = otelPropagatorType; }
     void setOtelMaxQueueSize(const unsigned int otelMaxQueueSize) { this->otelMaxQueueSize = otelMaxQueueSize; }
     void setOtelScheduledDelayMillis(const unsigned int otelScheduledDelayMillis) { this->otelScheduledDelayMillis = otelScheduledDelayMillis; }
     void setOtelExportTimeoutMillis(const unsigned int otelExportTimeoutMillis) { this->otelExportTimeoutMillis = otelExportTimeoutMillis; }
@@ -85,6 +87,7 @@ class TenantConfig
     std::string otelExporterOtlpHeaders;
     bool otelSslEnabled;
     std::string otelSslCertPath;
+    std::string otelPropagatorType;
 
     std::string otelProcessorType;
     std::string otelSamplerType;
@@ -108,6 +111,7 @@ inline std::ostream& operator<< (std::ostream &os, const otel::core::TenantConfi
         << "\n OtelExporterType                 " << config.getOtelExporterType()
         << "\n OtelExporterEndpoint             " << config.getOtelExporterEndpoint()
         << "\n OtelProcessorType                " << config.getOtelProcessorType()
+        << "\n OtelPropagatorType               " << config.getOtelPropagatorType()
         << "\n OtelSamplerType                  " << config.getOtelSamplerType()
         << "\n OtelSslEnabled                   " << config.getOtelSslEnabled()
         << "\n OtelSslCertPath                  " << config.getOtelSslCertPath()

instrumentation/otel-webserver-module/include/core/api/opentelemetry_ngx_api.h

@@ -25,7 +25,7 @@
 extern "C" {
 #endif
 
-const char* httpHeaders[] = {"traceparent", "tracestate"};
+const char* httpHeaders[] = {"x-b3-traceid", "x-b3-spanid", "x-b3-sampled", "traceparent", "tracestate"};
 const size_t headers_len = sizeof(httpHeaders)/sizeof(httpHeaders[0]);
 
 typedef struct{

instrumentation/otel-webserver-module/include/core/sdkwrapper/SdkHelperFactory.h

@@ -23,6 +23,7 @@
 #include <opentelemetry/nostd/shared_ptr.h>
 #include <opentelemetry/trace/tracer.h>
 #include <opentelemetry/trace/propagation/http_trace_context.h>
+#include <opentelemetry/trace/propagation/b3_propagator.h>
 #include <opentelemetry/context/propagation/text_map_propagator.h>
 #include <opentelemetry/exporters/ostream/span_exporter.h>
 #include <opentelemetry/sdk/trace/processor.h>

instrumentation/otel-webserver-module/src/core/api/ApiUtils.cpp

@@ -172,6 +172,7 @@ OTEL_SDK_STATUS_CODE ApiUtils::ReadSettingsFromReader(
     std::string otelSslCertPath;
     std::string otelLibraryName;
     std::string otelProcessorType;
+    std::string otelPropagatorType;
     std::string otelSamplerType;
 
     unsigned otelMaxQueueSize;
@@ -227,6 +228,9 @@ OTEL_SDK_STATUS_CODE ApiUtils::ReadSettingsFromReader(
     reader.ReadOptional(
         std::string(OTEL_SDK_ENV_OTEL_PROCESSOR_TYPE), otelProcessorType);
 
+    reader.ReadOptional(
+        std::string(OTEL_SDK_ENV_OTEL_PROPAGATOR_TYPE), otelPropagatorType);
+
     reader.ReadOptional(
         std::string(OTEL_SDK_ENV_OTEL_SAMPLER_TYPE), otelSamplerType);
 
@@ -268,6 +272,7 @@ OTEL_SDK_STATUS_CODE ApiUtils::ReadSettingsFromReader(
     tenantConfig.setOtelExporterOtlpHeaders(otelExporterOtlpHeaders);
     tenantConfig.setOtelLibraryName(otelLibraryName);
     tenantConfig.setOtelProcessorType(otelProcessorType);
+    tenantConfig.setOtelPropagatorType(otelPropagatorType);
     tenantConfig.setOtelSamplerType(otelSamplerType);
     tenantConfig.setOtelMaxQueueSize(otelMaxQueueSize);
     tenantConfig.setOtelScheduledDelayMillis(otelScheduledDelayMillis);

instrumentation/otel-webserver-module/src/core/api/opentelemetry_ngx_api.cpp

@@ -26,6 +26,12 @@
 otel::core::WSAgent wsAgent; // global variable for interface between Hooks and Core Logic
 std::unordered_set<std::string> requestHeadersToCapture;
 std::unordered_set<std::string> responseHeadersToCapture;
+std::unordered_map<std::string, std::string> propagationsHeaderParse = {
+    {"x-b3-traceid", "X-B3-TraceId"},
+    {"x-b3-spanid", "X-B3-SpanId"},
+    {"x-b3-sampled", "X-B3-Sampled"}
+};
+
 constexpr char delimiter = ',';
 
 void populatePayload(request_payload* req_payload, void* load)
@@ -44,7 +50,17 @@ void populatePayload(request_payload* req_payload, void* load)
     payload->set_client_ip(req_payload->client_ip);
 
     for(int i=0; i<req_payload->propagation_count; i++){
-        payload->set_http_headers(req_payload->propagation_headers[i].name, req_payload->propagation_headers[i].value);
+
+        std::string prop_header_lowercase(req_payload->propagation_headers[i].name);
+        for(auto &s : prop_header_lowercase ){
+            s = tolower(s);
+        }
+        if( propagationsHeaderParse.find(prop_header_lowercase) != propagationsHeaderParse.end() ){
+            payload->set_http_headers( propagationsHeaderParse[prop_header_lowercase] , req_payload->propagation_headers[i].value);
+        }
+        else{
+            payload->set_http_headers(req_payload->propagation_headers[i].name, req_payload->propagation_headers[i].value);
+        }
     }
 
     for (int i = 0; i < req_payload->request_headers_count; i++) {

instrumentation/otel-webserver-module/src/core/sdkwrapper/SdkHelperFactory.cpp

@@ -47,6 +47,7 @@ namespace {
   constexpr const char* ALWAYS_OFF_SAMPLER = "always_off";
   constexpr const char* PARENT_BASED_SAMPLER = "parent";
   constexpr const char* TRACE_ID_RATIO_BASED_SAMPLER = "trace_id_ratio";
+  constexpr const char* PROPAGATOR_TYPE_B3 = "b3";
 }
 
 SdkHelperFactory::SdkHelperFactory(
@@ -96,9 +97,19 @@ SdkHelperFactory::SdkHelperFactory(
         " and LibraryVersion " << libraryVersion);
 
     // Adding trace propagator
-    using MapHttpTraceCtx = opentelemetry::trace::propagation::HttpTraceContext;
-    mPropagators.push_back(
-        std::unique_ptr<MapHttpTraceCtx>(new MapHttpTraceCtx()));
+    auto propagatorType = config->getOtelPropagatorType();
+    LOG4CXX_INFO(mLogger, "Propagator type used to read and export Trace and Span info : " << propagatorType);
+    if(propagatorType == PROPAGATOR_TYPE_B3){
+        mPropagators.push_back(
+            std::unique_ptr<opentelemetry::trace::propagation::B3PropagatorMultiHeader>(
+                new opentelemetry::trace::propagation::B3PropagatorMultiHeader()));
+    }
+    else{
+        mPropagators.push_back(
+            std::unique_ptr<opentelemetry::trace::propagation::HttpTraceContext>(
+                new opentelemetry::trace::propagation::HttpTraceContext()));
+    }
+
 }
 
 OtelTracer SdkHelperFactory::GetTracer()

instrumentation/otel-webserver-module/src/core/sdkwrapper/SdkWrapper.cpp

@@ -26,8 +26,8 @@ namespace sdkwrapper {
 
 namespace {
 
-constexpr const char* TRACEPARENT_HEADER_NAME = "traceparent";
-constexpr const char* TRACESTATE_HEADER_NAME = "tracestate";
+constexpr const char* CARRIER_HEADER_NAME[] = {"X-B3-TraceId", "X-B3-SpanId", "X-B3-Sampled", "traceparent", "tracestate"};
+const size_t CARRIER_HEADER_LEN = sizeof(CARRIER_HEADER_NAME)/sizeof(CARRIER_HEADER_NAME[0]);
 
 } // anyonymous
 
@@ -80,8 +80,12 @@ void SdkWrapper::PopulatePropagationHeaders(
 	}
 
   // copy all relevant kv pairs into carrier
-  carrier[TRACEPARENT_HEADER_NAME] = otelCarrier.Get(TRACEPARENT_HEADER_NAME).data();
-  carrier[TRACESTATE_HEADER_NAME] = otelCarrier.Get(TRACESTATE_HEADER_NAME).data();
+  for (int i = 0; i < CARRIER_HEADER_LEN; i++) {
+	auto carrier_header = otelCarrier.Get(CARRIER_HEADER_NAME[i]).data();
+	if(carrier_header != ""){
+	  	carrier[CARRIER_HEADER_NAME[i]] = otelCarrier.Get(CARRIER_HEADER_NAME[i]).data();
+	}
+  }
 }
 
 trace::SpanKind SdkWrapper::GetTraceSpanKind(const SpanKind& kind)

instrumentation/otel-webserver-module/src/nginx/ngx_http_opentelemetry_module.c

@@ -399,6 +399,13 @@ static ngx_command_t ngx_http_opentelemetry_commands[] = {
       0,
       NULL},
 
+    { ngx_string("NginxModulePropagatorType"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_propagator,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_opentelemetry_loc_conf_t, nginxModulePropagatorType),
+      NULL},
+
     /* command termination */
     ngx_null_command
 };
@@ -508,6 +515,8 @@ static char* ngx_http_opentelemetry_merge_loc_conf(ngx_conf_t *cf, void *parent,
     ngx_conf_merge_str_value(conf->nginxModuleSegmentParameter, prev->nginxModuleSegmentParameter, "2");
     ngx_conf_merge_str_value(conf->nginxModuleRequestHeaders, prev->nginxModuleRequestHeaders, "");
     ngx_conf_merge_str_value(conf->nginxModuleResponseHeaders, prev->nginxModuleResponseHeaders, "");
+    ngx_conf_merge_str_value(conf->nginxModulePropagatorType, prev->nginxModulePropagatorType, "w3c");
+
 
     return NGX_CONF_OK;
 }
@@ -722,6 +731,24 @@ static char* ngx_otel_attributes_set(ngx_conf_t* cf, ngx_command_t* cmd, void* c
 
 }
 
+static char* ngx_conf_set_propagator(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    ngx_http_opentelemetry_loc_conf_t * my_conf=(ngx_http_opentelemetry_loc_conf_t *)conf;
+
+    ngx_str_t *value = cf->args->elts;
+    ngx_str_t elt;
+    if( !strcmp(value[1].data, "b3") || !strcmp(value[1].data, "B3") ){
+        elt.data = (u_char *)"b3";
+        elt.len = sizeof("b3") - 1;
+        my_conf->nginxModulePropagatorType = elt;
+    }
+    else{
+        elt.data = (u_char *)"w3c";
+        elt.len = sizeof("w3c") - 1;
+        my_conf->nginxModulePropagatorType = elt;
+    }
+    return NGX_CONF_OK;
+
+}
 static char* ngx_conf_ignore_path_set(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
     ngx_http_opentelemetry_loc_conf_t * my_conf=(ngx_http_opentelemetry_loc_conf_t *)conf;
 
@@ -863,68 +890,69 @@ static OTEL_SDK_STATUS_CODE otel_startInteraction(ngx_http_request_t* r, const c
 
 static void otel_payload_decorator(ngx_http_request_t* r, OTEL_SDK_ENV_RECORD* propagationHeaders, int count)
 {
-   ngx_list_part_t  *part;
-   ngx_table_elt_t  *header;
-   ngx_table_elt_t            *h;
-   ngx_http_header_t          *hh;
-   ngx_http_core_main_conf_t  *cmcf;
-   ngx_uint_t       nelts;
-
-   part = &r->headers_in.headers.part;
-   header = (ngx_table_elt_t*)part->elts;
-   nelts = part->nelts;
-
-   for(int i=0; i<count; i++){
-
-       int header_found=0;
-       for(ngx_uint_t j = 0; j<nelts; j++){
-           h = &header[j];
-           if(strcmp(httpHeaders[i], h->key.data)==0){
-
-               header_found=1;
-
-               if(h->key.data)
-                    ngx_pfree(r->pool, h->key.data);
-               if(h->value.data)
-                    ngx_pfree(r->pool, h->value.data);
-
-               break;
-           }
-       }
-       if(header_found==0)
-       {
-           h = ngx_list_push(&r->headers_in.headers);
-       }
-
-       if(h == NULL )
-            return;
+    ngx_list_part_t  *part;
+    ngx_table_elt_t  *header;
+    ngx_table_elt_t            *h;
+    ngx_http_header_t          *hh;
+    ngx_http_core_main_conf_t  *cmcf;
+    ngx_uint_t       nelts;
+
+    part = &r->headers_in.headers.part;
+    header = (ngx_table_elt_t*)part->elts;
+    nelts = part->nelts;
+
+    for(int i=0; i<count; i++){
+
+        int header_found=0;
+        int prop_index = -1;
+        for(ngx_uint_t j = 0; j<nelts; j++){
+            h = &header[j];
+            if(strcasecmp(propagationHeaders[i].name, h->key.data)==0){
+
+                header_found=1;
+
+                if(h->key.data)
+                        ngx_pfree(r->pool, h->key.data);
+                if(h->value.data)
+                        ngx_pfree(r->pool, h->value.data);
+
+                break;
+            }
+        }
+        if(header_found==0)
+        {
+            h = ngx_list_push(&r->headers_in.headers);
+        }
 
-       h->key.len = strlen(propagationHeaders[i].name);
-       h->key.data = ngx_pcalloc(r->pool, sizeof(char)*((h->key.len)+1));
-       strcpy(h->key.data, propagationHeaders[i].name);
+        if(h == NULL )
+                return;
 
-       ngx_writeTrace(r->connection->log, __func__, "Key : %s", propagationHeaders[i].name);
+        h->key.len = strlen(propagationHeaders[i].name);
+        h->key.data = ngx_pcalloc(r->pool, sizeof(char)*((h->key.len)+1));
+        strcpy(h->key.data, propagationHeaders[i].name);
 
-       h->hash = ngx_hash_key(h->key.data, h->key.len);
+        ngx_writeTrace(r->connection->log, __func__, "Key : %s", propagationHeaders[i].name);
 
-       h->value.len = strlen(propagationHeaders[i].value);
-       h->value.data = ngx_pcalloc(r->pool, sizeof(char)*((h->value.len)+1));
-       strcpy(h->value.data, propagationHeaders[i].value);
-       h->lowcase_key = h->key.data;
+        h->hash = ngx_hash_key(h->key.data, h->key.len);
 
-       cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module);
-       hh = ngx_hash_find(&cmcf->headers_in_hash, h->hash,h->lowcase_key, h->key.len);
-       if (hh && hh->handler(r, h, hh->offset) != NGX_OK) {
-           return;
-       }
+        h->value.len = strlen(propagationHeaders[i].value);
+        h->value.data = ngx_pcalloc(r->pool, sizeof(char)*((h->value.len)+1));
+        strcpy(h->value.data, propagationHeaders[i].value);
+        h->lowcase_key = h->key.data;
 
-       ngx_writeTrace(r->connection->log, __func__, "Value : %s", propagationHeaders[i].value);
+        cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module);
+        hh = ngx_hash_find(&cmcf->headers_in_hash, h->hash,h->lowcase_key, h->key.len);
+        if (hh && hh->handler(r, h, hh->offset) != NGX_OK) {
+            return;
+        }
 
-   }
-
-   ngx_http_otel_handles_t* ctx = ngx_http_get_module_ctx(r, ngx_http_opentelemetry_module);
-   ctx->propagationHeaders = propagationHeaders;
-   ctx->pheaderCount = count;
+        ngx_writeTrace(r->connection->log, __func__, "Value : %s", propagationHeaders[i].value);
+
+    }
+
+    ngx_http_otel_handles_t* ctx = ngx_http_get_module_ctx(r, ngx_http_opentelemetry_module);
+    ctx->propagationHeaders = propagationHeaders;
+    ctx->pheaderCount = count;
 }
 
 /*
@@ -1185,6 +1213,9 @@ static ngx_flag_t ngx_initialize_opentelemetry(ngx_http_request_t *r)
         env_config[ix].value = (const char*)(conf->nginxModuleSegmentParameter).data;
         ++ix;
 
+        env_config[ix].name = OTEL_SDK_ENV_OTEL_PROPAGATOR_TYPE;
+        env_config[ix].value = (const char*)(conf->nginxModulePropagatorType).data;
+        ++ix;
 
         // !!!
         // Remember to update the ngx_pcalloc call size if we add another parameter to the input array!
@@ -1814,7 +1845,7 @@ static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t*
 
         h = &header[j];
         for (int i = 0; i < headers_len && conf->nginxModuleTrustIncomingSpans ; i++) {
-
+            
             if (strcmp(h->key.data, httpHeaders[i]) == 0) {
                 req_payload->propagation_headers[propagation_headers_idx].name = httpHeaders[i];
                 req_payload->propagation_headers[propagation_headers_idx].value = (const char*)(h->value).data;