Distributed Tracing with elasticsearch

Linkerd General Discussion
, ,
1

Hi, I have been trying to integrate distributed tracing capabilities of linkerd with elk stack.

I have deployed linkerd and elk stack on kubernetes instance and I want the linkerd to send metrics to elk stack so that I can have the same tracing there.
Can you help me with this?

Here is how I try to do it unsuccessfully

apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
  name: release-linkerd-linkerd-es
  namespace: helm-releases
spec:
  repo: https://helm.linkerd.io/edge
  chart: linkerd-jaeger
  targetNamespace: linkerd
  version: 2024.4.4
  valuesContent: |-
    jaeger:
      enabled: false
    collector:
      config:
        exporters:
          logging:
            loglevel: info 
          otlp/elastic: 
            endpoint: "apm-apm-sample-apm-http.elk.svc.cluster.local:8200"
            tls:
              insecure: true
            headers:
              Authorization: "Bearer uJS7yDL2743C6QpD4a6A23QN"
        extensions:
          health_check: null
        processors:
          batch: null
          k8sattributes:
            auth_type: serviceAccount
            extract:
              labels:
                - from: pod
                  key: app.kubernetes.io/name
                  tag_name: kube_app_name
                - from: pod
                  key: app.kubernetes.io/instance
                  tag_name: kube_app_instance
                - from: pod
                  key: app.kubernetes.io/version
                  tag_name: kube_app_version
                - from: pod
                  key: app.kubernetes.io/component
                  tag_name: kube_app_component
                - from: pod
                  key: app.kubernetes.io/part-of
                  tag_name: kube_app_part_of
                - from: pod
                  key: app.kubernetes.io/managed-by
                  tag_name: kube_app_managed_by
              metadata:
                - k8s.pod.name
                - k8s.pod.uid
                - k8s.deployment.name
                - k8s.node.name
                - k8s.namespace.name
                - k8s.pod.start_time
                - k8s.replicaset.name
                - k8s.replicaset.uid
                - k8s.daemonset.name
                - k8s.daemonset.uid
                - k8s.job.name
                - k8s.job.uid
                - k8s.cronjob.name
                - k8s.statefulset.name
                - k8s.statefulset.uid
                - container.image.name
                - container.image.tag
                - container.id
                - k8s.container.name
                - container.image.name
                - container.image.tag
                - container.id
            pod_association:
              - sources:
                  - from: resource_attribute
                    name: k8s.pod.name
                  - from: resource_attribute
                    name: k8s.namespace.name
          resource:
            attributes:
              - action: insert
                from_attribute: host.name
                key: k8s.pod.name
              - action: insert
                from_attribute: linkerd.io/workload-ns
                key: k8s.namespace.name
        receivers:
          opencensus: null
        service:
          extensions:
            - health_check
          pipelines:
            traces:
              exporters:
                - logging
                - otlp/elastic
              processors:
                - resource
                - k8sattributes
                - batch
              receivers:
                - opencensus

Why doesn’t this work?

2

Do you have the “webhook” component configured (i.e., the jaeger-injector) so that it can activate span emissions from Linkerd proxies? In the config above I see the collector config, but we still need to tell Linkerd to emit the spans themselves.

3

Thank you for directing me, I will try to configure webhook next

4

Hi @Alen
I don’t understand I didnt change configuraation for receivers. Why should webhooks be changed?
I hsve changed only exporters.
Can you help me configure webhooks?
here is the full values yaml

clusterDomain: cluster.local
collector:
  UID: null
  command: /otelcol-contrib
  config:
    exporters:
      jaeger:
        endpoint: jaeger.${POD_NAMESPACE}:14250
        tls:
          insecure: true
      logging:
        loglevel: info
      otlp/elastic:
        endpoint: apmserver-elk-apm-http.elk.svc.cluster.local:8200
        headers:
          Authorization: Bearer G06sW87VC0m1Xp03TliE10dD
        tls:
          insecure: true
    extensions:
      health_check: null
    processors:
      batch: null
      k8sattributes:
        auth_type: serviceAccount
        extract:
          labels:
            - from: pod
              key: app.kubernetes.io/name
              tag_name: kube_app_name
            - from: pod
              key: app.kubernetes.io/instance
              tag_name: kube_app_instance
            - from: pod
              key: app.kubernetes.io/version
              tag_name: kube_app_version
            - from: pod
              key: app.kubernetes.io/component
              tag_name: kube_app_component
            - from: pod
              key: app.kubernetes.io/part-of
              tag_name: kube_app_part_of
            - from: pod
              key: app.kubernetes.io/managed-by
              tag_name: kube_app_managed_by
          metadata:
            - k8s.pod.name
            - k8s.pod.uid
            - k8s.deployment.name
            - k8s.node.name
            - k8s.namespace.name
            - k8s.pod.start_time
            - k8s.replicaset.name
            - k8s.replicaset.uid
            - k8s.daemonset.name
            - k8s.daemonset.uid
            - k8s.job.name
            - k8s.job.uid
            - k8s.cronjob.name
            - k8s.statefulset.name
            - k8s.statefulset.uid
            - container.image.name
            - container.image.tag
            - container.id
            - k8s.container.name
            - container.image.name
            - container.image.tag
            - container.id
        pod_association:
          - sources:
              - from: resource_attribute
                name: k8s.pod.name
              - from: resource_attribute
                name: k8s.namespace.name
      resource:
        attributes:
          - action: insert
            from_attribute: host.name
            key: k8s.pod.name
          - action: insert
            from_attribute: linkerd.io/workload-ns
            key: k8s.namespace.name
    receivers:
      opencensus: null
    service:
      extensions:
        - health_check
      pipelines:
        traces:
          exporters:
            - logging
            - otlp/elastic
          processors:
            - resource
            - k8sattributes
            - batch
          receivers:
            - opencensus
  enabled: true
  env: []
  image:
    name: otel/opentelemetry-collector-contrib
    pullPolicy: ''
    version: 0.83.0
  nodeSelector:
    kubernetes.io/os: linux
  replicas: 1
  resources:
    cpu:
      limit: null
      request: null
    ephemeral-storage:
      limit: ''
      request: ''
    memory:
      limit: null
      request: null
  tolerations: null
commonLabels: {}
createNamespaceMetadataJob: true
defaultUID: 2103
enablePSP: false
enablePodAntiAffinity: false
imagePullSecrets: []
jaeger:
  UID: null
  args:
    - '--query.base-path=/jaeger'
  enabled: false
  image:
    name: jaegertracing/all-in-one
    pullPolicy: ''
    version: 1.31
  nodeSelector:
    kubernetes.io/os: linux
  resources:
    cpu:
      limit: null
      request: null
    ephemeral-storage:
      limit: ''
      request: ''
    memory:
      limit: null
      request: null
  tolerations: null
linkerdNamespace: linkerd
linkerdVersion: edge-24.4.4
namespaceMetadata:
  image:
    name: extension-init
    pullPolicy: ''
    registry: cr.l5d.io/linkerd
    tag: v0.1.0
  nodeSelector:
    kubernetes.io/os: linux
  tolerations: null
nodeSelector:
  kubernetes.io/os: linux
podLabels: {}
tolerations: null
webhook:
  UID: null
  caBundle: ''
  collectorSvcAccount: collector
  collectorSvcAddr: collector.linkerd-jaeger:55678
  crtPEM: ''
  externalSecret: false
  failurePolicy: Ignore
  image:
    name: cr.l5d.io/linkerd/jaeger-webhook
    pullPolicy: ''
    version: edge-24.4.4
  injectCaFrom: ''
  injectCaFromSecret: ''
  keyPEM: ''
  logLevel: info
  namespaceSelector:
    matchExpressions:
      - key: kubernetes.io/metadata.name
        operator: NotIn
        values:
          - kube-system
  nodeSelector:
    kubernetes.io/os: linux
  objectSelector: null
  replicas: 1
  resources:
    cpu:
      limit: null
      request: null
    memory:
      limit: null
      request: null
  tolerations: null

``