cd ~
curl -LJO https://github.com/prometheus/prometheus/releases/download/v2.47.1/prometheus-2.47.1.linux-amd64.tar.gz
tar -xvzf prometheus-2.47.1.linux-amd64.tar.gz --directory ~

ln -sf prometheus-2.47.1.linux-amd64 prometheus

# Adjustments:
# Adjust the Controller username and password
# Adjust the Controller targets and metrics_path
# Local Controller:
# - metrics_path: '/uc/resources/metrics'
# - target: 'wiesloch:8080'
# Cloud Controller:
# - metrics_path: '/resources/metrics'
# - target: 'uc.companya.cloud'

# vi ~/prometheus/prometheus.yml

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

  - job_name: 'OTelCollector'
    static_configs:
      - targets: ["127.0.0.1:8881"]
        #- job_name: "OTelCollector"
        #static_configs:
        #- targets: ["127.0.0.1:8888"]

  - job_name: 'prometheus'
    metrics_path: /metrics
    static_configs:
    - targets:
      - localhost:9090
      
  - job_name: 'controller'
    basic_auth:
       username: 'observ'
       password: 'observ'
    metrics_path: '/uc/resources/metrics'  # for cloud controller without /uc/: /resources/metrics
    static_configs:
      - targets:
              - 'wiesloch:8080'  # Use the correct hostname or IP address and port of your web service.

# vi ~/start_prometheus.sh

#! /bin/bash

timestamp=$(date +%Y%m%d%H%M%S)
cd ~/prometheus
nohup ./prometheus --config.file=prometheus.yml --log.level=debug 0</dev/null >prometheus_$timestamp.out 2>prometheus_$timestamp.err &
sleep 3
echo "Started $(pidof prometheus)"

# vi ~/stop_prometheus.sh

#! /bin/bash

pid=$(pidof prometheus)
if [ -n "$pid" ]; then
    kill $pid
    echo "Stopped $pid"
else
    echo "No pid found"
fi

cd ~
chmod +x start_prometheus.sh
chmod +x stop_prometheus.sh

./start_prometheus.sh
./stop_prometheus.sh

Log Files:
~/prometheus/prometheus_<YYYYMMDDHHMMSS>.err
~/prometheus/prometheus_<YYYYMMDDHHMMSS>.out
URL:
http://wiesloch:9090
Port:
ss -tuln | grep 9090
=> tcp    LISTEN  0       128                       *:9090                 *:*
Process:
ps aux | grep prometheus
=> stonebr+  38093  0.1  0.4 1316792 76608 pts/0   Sl   16:33   0:00 ./prometheus --config.file=prometheus.yml --log.level=debug

cd ~
curl -LJO https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.82.0/otelcol-contrib_0.82.0_linux_amd64.tar.gz
mkdir ~/otelcol
tar -xvzf otelcol-contrib_0.82.0_linux_amd64.tar.gz --directory ~/otelcol

# verify that the http port 4318 (default) is the same then in the omss.conf and uags.conf

# vi ~/otelcol/otel-collector-config.yaml

receivers:
  otlp:
    protocols:
      http:
        #tls:
        #cert_file: ue-cert.crt
        #key_file: ue-cert.key
        #endpoint: 0.0.0.0:4318
      grpc:
        #endpoint: 0.0.0.0:4317
exporters:
  logging:
    verbosity: detailed
  prometheus:
    endpoint: 0.0.0.0:8881
      #tls:
      #ca_file: "/path/to/ca.pem"
      #cert_file: "/path/to/cert.pem"
      #key_file: "/path/to/key.pem"
      #namespace: UAgent
      #const_labels:
      #label1: value1
      #"another label": spaced value
      #send_timestamps: true
      #metric_expiration: 180m
      #enable_open_metrics: true
      #add_metric_suffixes: false
    resource_to_telemetry_conversion:
      enabled: true
processors:
  batch:
service:
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

# vi ~/start_otelcol.sh

#! /bin/bash

timestamp=$(date +%Y%m%d%H%M%S)
cd ~/otelcol
nohup ./otelcol-contrib --config=otel-collector-config.yaml 0</dev/null >otelcol_$timestamp.out 2>otelcol_$timestamp.err &
sleep 3
echo "Started $(pidof otelcol-contrib)"

# vi ~/stop_otelcol.sh

#! /bin/bash

pid=$(pidof otelcol-contrib)
if [ -n "$pid" ]; then
    kill $pid
    echo "Stopped $pid"
else
    echo "No pid found"
fi

cd ~
chmod +x start_otelcol.sh
chmod +x stop_otelcol.sh

./start_otelcol.sh
./stop_otelcol.sh

Log Files:
~/otelcol/otelcol_<YYYYMMDDHHMMSS>.err
~/otelcol/otelcol_<YYYYMMDDHHMMSS>.out

Process:
ps aux | grep otelcol-contrib
=> stonebr+  38898  0.2  0.7 913032 129860 pts/0   Sl   16:41   0:00 ./otelcol-contrib --config=otel-collector-config.yaml

cd ~
curl -O https://dl.grafana.com/enterprise/release/grafana-enterprise-10.1.4.linux-amd64.tar.gz

tar -xvzf grafana-enterprise-10.1.4.linux-amd64.tar.gz --directory ~

ln -sf grafana-10.1.4 ~/grafana

# vi ~/start_grafana.sh

#! /bin/bash

timestamp=$(date +%Y%m%d%H%M%S)
cd ~/grafana 
nohup bin/grafana server 0</dev/null >grafana_$timestamp.out 2>grafana_$timestamp.err &
sleep 3
echo "Started $(pidof grafana)"

# vi ~/stop_grafana.sh

#! /bin/bash

pid=$(pidof grafana)
if [ -n "$pid" ]; then
    kill $pid
    echo "Stopped $pid"
else
    echo "No pid found"
fi

cd ~
chmod +x start_grafana.sh
chmod +x stop_grafana.sh

./stop_grafana.sh
./start_garfana.sh

Log Files:
~/ grafana/grafana_<YYYYMMDDHHMMSS>.err
~/ grafana/grafana_<YYYYMMDDHHMMSS>.out
URL:
http://192.168.88.17:3000/
user:admin
pass:admin

# Add the following to uags.conf
# Adjust the sample IP "192.168.88.17" Address to your IP or hostname

sudo vi /etc/universal/uags.conf

otel_export_metrics YES
otel_service_name agt_wiesloch_uag
otel_metrics_endpoint http://192.168.88.17:4318

# Add the following to omss.conf

sudo vi /etc/universal/omss.conf

otel_export_metrics YES
otel_service_name agt_wiesloch_oms
otel_metrics_endpoint http://192.168.88.17:4318

# Note: Agents push metrics via http ( not grpc ) port 4318

# restart/Stop Universal Agent
sudo /opt/universal/ubroker/ubrokerd restart
sudo /opt/universal/ubroker/ubrokerd stop

Log Files:
# UAG 
sudo vi /var/opt/universal/uag/logs/agent.log
# Broker
sudo vi /var/opt/universal/log/unv.log

# enable metrics for UC Controller in uc.properties file
# Adjust the vi command to point to your uc.properties file

# sudo vi /usr/share/apache-tomcat-9.0.80/conf/uc.properties

# Metrics uses http protocol:
uc.otel.exporter.otlp.metrics.endpoint http://localhost:4317

# stop/start Tomcat ( adjust according to your environment )
sudo /usr/share/apache-tomcat-9.0.80/bin/shutdown.sh
sudo /usr/share/apache-tomcat-9.0.80/bin/startup.sh

Optionally
sudo /etc/init.d/uac_tomcat stop
sudo /etc/init.d/uac_tomcat start

Log Files:
# Adjust to the location of your Tomcat directory
sudo cat /usr/share/apache-tomcat-9.0.80/uc_logs/uc.log

# Creating a Self-signed Certificate for Grafana TLS connection

documentation link:
https://grafana.com/docs/grafana/latest/setup-grafana/set-up-https/

cd ~/grafana
# generate private key
sudo openssl genrsa -out ~/grafana/grafana.key 2048

# generate certificate using the private key
sudo openssl req -new -key ~/grafana/grafana.key -out ~/grafana/grafana.csr

# check
ls -ltr ~/grafana
-rw-------  1 root        root   1679 Nov 10 09:24 grafana.key
-rw-r--r--  1 root        root   1070 Nov 10 09:28 grafana.csr

# create self-sign the certificate with the private key, for a period of validity of 720 days:
sudo openssl x509 -req -days 720 -in ~/grafana/grafana.csr -signkey ~/grafana/grafana.key -out ~/grafana/grafana.crt

# set the appropriate permissions for the files
sudo chown stonebranch ~/grafana/grafana.crt
sudo chown stonebranch ~/grafana/grafana.key
sudo chmod 400 ~/grafana/grafana.key ~/grafana/grafana.crt

# adjust grafana ini file defaults.ini
vi ~/grafana/conf/defaults.ini

[server]
protocol = https
http_port = 3000
domain = wiesloch
cert_file = /home/stonebranch/grafana/grafana.crt
cert_key = /home/stonebranch/grafana/grafana.key

# re-start grafana.
~/start_grafana.sh



# Check URL accessible via HTTPS
URL:
https://192.168.88.17:3000/
Login: admin/admin

# Using Self-signed Certificates in Firefox

Type: about:preferences#privacy and go to: Privacy & Security -> View Certificates
(go to Settings -> Privacy & Security -> View Certificates)
choose Servers Tab and click Add Exception
fill in the https URL: https://192.168.88.17:3000 and click Get Certificate
click Confirm Security Exception

cd ~
curl -LJO https://github.com/jaegertracing/jaeger/releases/download/v1.47.0/jaeger-1.47.0-linux-amd64.tar.gz
tar -xvzf jaeger-1.47.0-linux-amd64.tar.gz --directory ~
# create Softlink
ln -sf jaeger-1.47.0-linux-amd64 ~/jaeger

# vi ~/jaeger/jaeger-config.yaml

collector:
  otlp:
    grpc:
      host-port: "14317"
    http:
      host-port: "14318"
http_server:
  base_path: /jaeger

# vi ~/start_jaeger.sh

#! /bin/bash
timestamp=$(date +%Y%m%d%H%M%S)
#export JAEGER_SERVICE_NAME=jenkins-tst_traces
export SPAN_STORAGE_TYPE="elasticsearch"
cd ~/jaeger
nohup ./jaeger-all-in-one --config-file /home/stonebranch/jaeger/jaeger-config.yaml 0</dev/null >jaeger-all-in-one_$timestamp.out 2>jaeger-all-in-one_$timestamp.err &
sleep 3
echo "Started $(pidof jaeger-all-in-one)"

# vi ~/stop_jaeger.sh

#! /bin/bash
pid=$(pidof jaeger-all-in-one)
if [ -n "$pid" ]; then
    kill $pid
    echo "Stopped $pid"
else
    echo "No pid found"
fi

cd ~
./start_jaeger.sh
./stop_jaeger.sh

Log Files:
~/jaeger/jaeger-all-in-one_<YYYYMMDDHHMMSS>.err
~/jaeger/jaeger-all-in-one_<YYYYMMDDHHMMSS>.out

Port:
sudo ss -tulpn | grep 4317
=> tcp    LISTEN  0       128                       *:14317                *:*      users:(("jaeger-all-in-o",pid=10251,fd=11))

URL:
http://wiesloch:16686/

# vi ~/otelcol/otel-collector-config.yaml

# Add Jaeger in exporters section
# Add Jaeger for traces

receivers:
  otlp:
    protocols:
      http:
        #tls:
        #cert_file: ue-cert.crt
        #key_file: ue-cert.key
        #endpoint: 0.0.0.0:4318
      grpc:
        #endpoint: 0.0.0.0:4317
exporters:
  logging:
    verbosity: detailed
  prometheus:
    endpoint: 0.0.0.0:8881
      #tls:
      #ca_file: "/path/to/ca.pem"
      #cert_file: "/path/to/cert.pem"
      #key_file: "/path/to/key.pem"
      #namespace: UAgent
      #const_labels:
      #label1: value1
      #"another label": spaced value
      #send_timestamps: true
      #metric_expiration: 180m
      #enable_open_metrics: true
      #add_metric_suffixes: false
    resource_to_telemetry_conversion:
      enabled: true
  jaeger:
    endpoint: localhost:14250
    tls:
      insecure: true
processors:
  batch:
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [jaeger]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

cd ~
chmod +x ./start_otelcol.sh
chmod +x ./stop_otelcol.sh

./start_otelcol.sh
./stop_otelcol.sh

Log Files:
~/otelcol/otelcol_<YYYYMMDDHHMMSS>.err
~/otelcol/otelcol_<YYYYMMDDHHMMSS>.out

# stop tomcat ( adjust according to your environment )
sudo /usr/share/apache-tomcat-9.0.80/bin/shutdown.sh

# enable traces (grpc protocol) for Universal Controller
sudo vi /usr/share/apache-tomcat-9.0.80/conf/uc.properties
uc.otel.exporter.otlp.traces.endpoint http://localhost:4317

# start tomcat ( adjust according to your environment )
sudo /usr/share/apache-tomcat-9.0.80/bin/startup.sh

# stop/start Tomcat ( adjust according to your environment )
sudo /usr/share/apache-tomcat-9.0.80/bin/shutdown.sh
sudo /usr/share/apache-tomcat-9.0.80/bin/startup.sh

Optionally:
sudo /etc/init.d/uac_tomcat stop
sudo /etc/init.d/uac_tomcat start

Log Files:
# Adjust to the location of your Tomcat directory
sudo cat /usr/share/apache-tomcat-9.0.80/uc_logs/uc.log

vi ~/grafana/conf/default.ini

#################################### Security ############################
[security]
...
# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.

allow_embedding = true
...
# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled"
cookie_samesite = disabled
...

# Add the following to:
# uags.conf and omss.conf

sudo vi /etc/universal/uags.conf
sudo vi /etc/universal/omss.conf

otel_enable_tracing YES
otel_trace_endpoint http://192.168.88.17:4318

# Restart the Agent
sudo /opt/universal/ubroker/ubrokerd restart

# Start/Stop Universal Agent
sudo /opt/unversal/ubroker/ubrokerd start
sudo /opt/unversal/ubroker/ubrokerd stop

Log Files:
# UAG 
sudo vi /var/opt/universal/uag/logs/agent.log
# Broker
sudo vi /var/opt/universal/log/unv.log

cd ~
./start_elasticsearch.sh
./start_jaeger.sh 
./start_prometheus.sh 
./start_otelcol.sh
./start_grafana.sh

cd ~
./stop_grafana.sh
./stop_otelcol.sh
./stop_prometheus.sh
./stop_jaeger.sh 
./stop_elasticsearch.sh

vi  ~/start_apm.sh

#! /bin/bash

if [ $# -eq 0 ]; then
	module_list='elasticsearch jaeger prometheus otelcol grafana'
else
	module_list=$@
fi

i=0
for module in $module_list
do
	i=$((i + 1))
	echo -n "Starting - $i: $module ... "
	./start_$module.sh
	#PID=$!
	#wait $PID
	#echo " ok (PID: $PID)"
done

vi ~/stop_apm.sh

#! /bin/bash

if [ $# -eq 0 ]; then
	module_list='grafana otelcol prometheus jaeger elasticsearch'
else
	module_list=$@
fi

i=0
for module in $module_list
do
	i=$((i + 1))
	echo -n "Stopping - $i: $module ... "
	./stop_$module.sh
	#PID=$!
	#wait $PID
	#echo " ok (PID: $PID)"
done

start/stop all applications:
./start_apm.sh
./stop_apm.sh

{
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "This Widget shows the number of Agents connected and has an indicator for the upper limit of how many Agents can connect to the controller. The upper limit depends on the number of licenses the controller owns.",
  "fieldConfig": {
    "defaults": {
      "custom": {
        "drawStyle": "line",
        "lineInterpolation": "linear",
        "barAlignment": 0,
        "lineWidth": 1,
        "fillOpacity": 58,
        "gradientMode": "none",
        "spanNulls": true,
        "insertNulls": false,
        "showPoints": "never",
        "pointSize": 5,
        "stacking": {
          "mode": "none",
          "group": "A"
        },
        "axisPlacement": "auto",
        "axisLabel": "",
        "axisColorMode": "series",
        "scaleDistribution": {
          "type": "linear"
        },
        "axisCenteredZero": false,
        "hideFrom": {
          "tooltip": false,
          "viz": false,
          "legend": false
        },
        "thresholdsStyle": {
          "mode": "off"
        },
        "axisGridShow": false,
        "axisSoftMin": 0,
        "lineStyle": {
          "fill": "solid"
        }
      },
      "color": {
        "mode": "palette-classic"
      },
      "mappings": [],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          },
          {
            "color": "red",
            "value": 80
          }
        ]
      },
      "decimals": 0
    },
    "overrides": [
      {
        "matcher": {
          "id": "byFrameRefID",
          "options": "Maximum Agents"
        },
        "properties": [
          {
            "id": "displayName",
            "value": "Max Number of Agents"
          },
          {
            "id": "color",
            "value": {
              "fixedColor": "dark-red",
              "mode": "fixed",
              "seriesBy": "max"
            }
          },
          {
            "id": "custom.fillOpacity",
            "value": 0
          }
        ]
      },
      {
        "matcher": {
          "id": "byFrameRefID",
          "options": "#Agents connected"
        },
        "properties": [
          {
            "id": "displayName",
            "value": "Total Number of used Agents"
          },
          {
            "id": "color",
            "value": {
              "fixedColor": "green",
              "mode": "shades"
            }
          }
        ]
      }
    ]
  },
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 18,
    "y": 29
  },
  "id": 11,
  "options": {
    "tooltip": {
      "mode": "single",
      "sort": "none"
    },
    "legend": {
      "showLegend": true,
      "displayMode": "list",
      "placement": "bottom",
      "calcs": []
    }
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "uc_license_agents_distributed_max",
      "fullMetaSearch": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "Maximum Agents",
      "useBackend": false
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "uc_license_agents_distributed_used",
      "fullMetaSearch": false,
      "hide": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "#Agents connected",
      "useBackend": false
    }
  ],
  "title": "#Agents connected",
  "type": "timeseries"
}

{
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "OMS Server Status shown in a \"Status History\" graph. Depending on the number of OMS server connected the graph changes to represent them. Every addition of an OMS server needs a new query for overriding the name.",
  "fieldConfig": {
    "defaults": {
      "mappings": [
        {
          "options": {
            "0": {
              "color": "red",
              "index": 0,
              "text": "Offline"
            },
            "1": {
              "color": "green",
              "index": 1,
              "text": "Online"
            },
            "-1": {
              "color": "yellow",
              "index": 2,
              "text": "In doubt"
            }
          },
          "type": "value"
        }
      ],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          }
        ]
      },
      "unit": "short"
    },
    "overrides": []
  },
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 0,
    "y": 29
  },
  "id": 1,
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "lastNotNull"
      ],
      "fields": ""
    },
    "orientation": "auto",
    "textMode": "auto",
    "colorMode": "background",
    "graphMode": "none",
    "justifyMode": "auto"
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(instance) (uc_oms_server_status)",
      "fullMetaSearch": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "OMS Server Status",
      "useBackend": false
    }
  ],
  "title": "OMS Server Status",
  "type": "stat"
}

{
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "Widget that shows how many Clients are connecting to an OMS server. It will count the connections from agents and controller that connect to the OMS server",
  "fieldConfig": {
    "defaults": {
      "mappings": [
        {
          "options": {
            "0": {
              "color": "light-red",
              "index": 0,
              "text": "No active connections"
            }
          },
          "type": "value"
        }
      ],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          }
        ]
      },
      "color": {
        "mode": "thresholds"
      }
    },
    "overrides": []
  },
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 12,
    "y": 37
  },
  "id": 4,
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "lastNotNull"
      ],
      "fields": ""
    },
    "orientation": "auto",
    "textMode": "auto",
    "colorMode": "value",
    "graphMode": "area",
    "justifyMode": "auto"
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(instance) (ua_active_connections)",
      "fullMetaSearch": false,
      "hide": false,
      "includeNullMetadata": false,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "A",
      "useBackend": false
    }
  ],
  "title": "OMS Client connections",
  "type": "stat"
}

{
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 18,
    "y": 20
  },
  "id": 20,
  "title": "Tasks started in the last 24h",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(task_type) (increase(uc_history_total[24h]))",
      "fullMetaSearch": false,
      "hide": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "Task history 24h",
      "useBackend": false
    }
  ],
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "lastNotNull"
      ],
      "fields": ""
    },
    "orientation": "auto",
    "textMode": "value_and_name",
    "colorMode": "background",
    "graphMode": "area",
    "justifyMode": "center",
    "text": {
      "titleSize": 16,
      "valueSize": 16
    }
  },
  "fieldConfig": {
    "defaults": {
      "mappings": [
        {
          "options": {
            "0": {
              "color": "text",
              "index": 0,
              "text": "No Tasks yet"
            }
          },
          "type": "value"
        }
      ],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          }
        ]
      },
      "decimals": 0,
      "unit": "short"
    },
    "overrides": []
  },
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "How many tasks started in the last 24h\n",
  "pluginVersion": "10.1.4",
  "type": "stat"
}

{
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "Shows the duration of tasks in the last given time period.",
  "fieldConfig": {
    "defaults": {
      "custom": {
        "hideFrom": {
          "tooltip": false,
          "viz": false,
          "legend": false
        }
      },
      "color": {
        "mode": "palette-classic"
      },
      "mappings": [],
      "decimals": 1,
      "unit": "dtdurations"
    },
    "overrides": []
  },
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 12,
    "y": 12
  },
  "id": 23,
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "lastNotNull"
      ],
      "fields": ""
    },
    "pieType": "pie",
    "tooltip": {
      "mode": "single",
      "sort": "none"
    },
    "legend": {
      "showLegend": true,
      "displayMode": "list",
      "placement": "right",
      "values": [
        "percent"
      ]
    }
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(task_type) (changes(uc_task_instance_duration_seconds_bucket[24h]))",
      "fullMetaSearch": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "Task durations",
      "useBackend": false
    }
  ],
  "title": "Task duration split",
  "type": "piechart"
}

{
  "datasource": {
    "type": "prometheus",
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
  },
  "description": "Pie chart which shows the percentage of \"Late Tasks\" in reference to the total amount of tasks (Last 1h in this example; Linux Tasks in this example)",
  "fieldConfig": {
    "defaults": {
      "custom": {
        "hideFrom": {
          "tooltip": false,
          "viz": false,
          "legend": false
        }
      },
      "color": {
        "mode": "palette-classic"
      },
      "mappings": [
        {
          "options": {
            "0": {
              "index": 0,
              "text": "None"
            }
          },
          "type": "value"
        }
      ],
      "decimals": 0,
      "noValue": "-"
    },
    "overrides": [
      {
        "matcher": {
          "id": "byFrameRefID",
          "options": "Late_Finished_Tasks"
        },
        "properties": [
          {
            "id": "color",
            "value": {
              "fixedColor": "yellow",
              "mode": "fixed"
            }
          },
          {
            "id": "displayName",
            "value": "Late Finish Tasks"
          }
        ]
      },
      {
        "matcher": {
          "id": "byFrameRefID",
          "options": "Total_Linux_Tasks"
        },
        "properties": [
          {
            "id": "color",
            "value": {
              "fixedColor": "green",
              "mode": "fixed"
            }
          }
        ]
      }
    ]
  },
  "gridPos": {
    "h": 8,
    "w": 6,
    "x": 12,
    "y": 20
  },
  "id": 22,
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "lastNotNull"
      ],
      "fields": ""
    },
    "pieType": "pie",
    "tooltip": {
      "mode": "single",
      "sort": "none"
    },
    "legend": {
      "showLegend": true,
      "displayMode": "list",
      "placement": "right",
      "values": [
        "percent"
      ]
    },
    "displayLabels": [
      "percent",
      "name"
    ]
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(task_type) (delta(uc_task_instance_late_finish_total[1h]))",
      "fullMetaSearch": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "Late_Finished_Tasks",
      "useBackend": false
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "builder",
      "expr": "sum by(task_type) (delta(uc_history_total[1h])) - sum by(task_type) (delta(uc_task_instance_late_finish_total[1h]))",
      "fullMetaSearch": false,
      "hide": false,
      "includeNullMetadata": true,
      "instant": false,
      "legendFormat": "__auto",
      "range": true,
      "refId": "Total_Linux_Tasks",
      "useBackend": false
    }
  ],
  "title": "Task Instance Late Finish",
  "type": "piechart"
}

{
  "datasource": {
    "type": "jaeger",
    "uid": "ba9176e4-0b3b-437c-ab29-045d734b5b63"
  },
  "description": "This Widget displays all the traces coming from the controller and agent and displaying their duration. Hovering over a trace will give more information about the trace.",
  "fieldConfig": {
    "defaults": {
      "custom": {
        "lineWidth": 1,
        "fillOpacity": 80,
        "gradientMode": "hue",
        "axisPlacement": "left",
        "axisLabel": "",
        "axisColorMode": "series",
        "scaleDistribution": {
          "type": "log",
          "log": 10
        },
        "axisCenteredZero": false,
        "hideFrom": {
          "tooltip": false,
          "viz": false,
          "legend": false
        },
        "thresholdsStyle": {
          "mode": "off"
        }
      },
      "color": {
        "mode": "thresholds"
      },
      "mappings": [],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          }
        ]
      },
      "unit": "s"
    },
    "overrides": []
  },
  "gridPos": {
    "h": 10,
    "w": 12,
    "x": 0,
    "y": 17
  },
  "id": 37,
  "links": [],
  "options": {
    "orientation": "auto",
    "xTickLabelRotation": 0,
    "xTickLabelSpacing": 300,
    "showValue": "auto",
    "stacking": "none",
    "groupWidth": 0.7,
    "barWidth": 0.97,
    "barRadius": 0,
    "fullHighlight": false,
    "tooltip": {
      "mode": "multi",
      "sort": "none"
    },
    "legend": {
      "showLegend": false,
      "displayMode": "list",
      "placement": "bottom",
      "calcs": []
    },
    "xField": "Start time"
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "jaeger",
        "uid": "ba9176e4-0b3b-437c-ab29-045d734b5b63"
      },
      "queryType": "search",
      "refId": "Traces",
      "service": "controller"
    }
  ],
  "title": "Trace Log",
  "transformations": [
    {
      "id": "sortBy",
      "options": {
        "fields": {},
        "sort": [
          {
            "field": "Start time",
            "desc": false
          }
        ]
      }
    }
  ],
  "type": "barchart"
}

{
  "datasource": {
    "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949",
    "type": "prometheus"
  },
  "description": "Average Duration is computed within the Time Period selected. The Max value of it is displayed",
  "fieldConfig": {
    "defaults": {
      "mappings": [],
      "thresholds": {
        "mode": "absolute",
        "steps": [
          {
            "color": "green",
            "value": null
          }
        ]
      },
      "color": {
        "fixedColor": "super-light-yellow",
        "mode": "fixed"
      },
      "unit": "s"
    },
    "overrides": [
      {
        "matcher": {
          "id": "byName",
          "options": "Value"
        },
        "properties": [
          {
            "id": "color",
            "value": {
              "fixedColor": "super-light-yellow",
              "mode": "fixed"
            }
          }
        ]
      }
    ]
  },
  "gridPos": {
    "h": 4,
    "w": 4,
    "x": 0,
    "y": 1
  },
  "id": 20,
  "interval": "30",
  "options": {
    "reduceOptions": {
      "values": false,
      "calcs": [
        "max"
      ],
      "fields": ""
    },
    "orientation": "auto",
    "textMode": "auto",
    "colorMode": "value",
    "graphMode": "area",
    "justifyMode": "auto"
  },
  "pluginVersion": "10.1.4",
  "targets": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "a65085b5-82cf-490b-a6cb-c01306f4a949"
      },
      "disableTextWrap": false,
      "editorMode": "code",
      "expr": "sum(increase(ue_cdt_rclone_duration_sum{universal_extension_name=\"ue-cloud-dt\"}[24h])) / sum(increase(ue_cdt_rclone_duration_count{universal_extension_name=\"ue-cloud-dt\"}[24h]))",
      "fullMetaSearch": false,
      "includeNullMetadata": true,
      "instant": false,
      "interval": "",
      "legendFormat": "{{label_name}}",
      "range": true,
      "refId": "Average Speed Over Time (MB/s)",
      "useBackend": false
    }
  ],
  "title": "Max of Average duration",
  "type": "stat"
}