From cae42653bfc9b81ef0ccda93746243dd67713489 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 20 May 2024 15:47:23 -0400 Subject: Add recording * Always output awx logs to a file via otel * That log file can always be later replayed into a product that supports otlp at a later date. * Useful when you find a problem that you need a time series DB to help find and solve. * Useful if a community member or customer has a problem where a time series db would be helpful. You can take a "remote" users log and replay it locally for analysis. --- licenses/pyzstd.txt | 29 ++++++++++++++++++++++ requirements/requirements.in | 1 + requirements/requirements.txt | 2 ++ .../roles/sources/templates/docker-compose.yml.j2 | 7 +----- tools/otel/otel-collector-config.yaml | 13 +++++++++- 5 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 licenses/pyzstd.txt diff --git a/licenses/pyzstd.txt b/licenses/pyzstd.txt new file mode 100644 index 0000000000..e69dae4dd4 --- /dev/null +++ b/licenses/pyzstd.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020-present, Ma Lin +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/requirements/requirements.in b/requirements/requirements.in index fd1efc7734..5e25a766da 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -55,6 +55,7 @@ python-dsv-sdk>=1.0.4 python-tss-sdk>=1.2.1 python-ldap pyyaml>=6.0.1 +pyzstd receptorctl social-auth-core[openidconnect]==4.4.2 # see UPGRADE BLOCKERs social-auth-app-django==5.4.0 # see UPGRADE BLOCKERs diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f1516afce5..502c4a1b52 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -446,6 +446,8 @@ pyyaml==6.0.1 # djangorestframework-yaml # kubernetes # receptorctl +pyzstd==0.15.10 + # via -r /awx_devel/requirements/requirements.in receptorctl==1.4.4 # via -r /awx_devel/requirements/requirements.in redis==5.0.1 diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index c6a0b4ed90..86f84523a3 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -283,8 +283,7 @@ services: - "55679:55679" # zpages http://localhost:55679/debug/servicez /tracez volumes: - "../../otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml" - depends_on: - - loki + - "../../otel/awx-logs:/awx-logs/" {% endif %} {% if enable_loki|bool %} loki: @@ -298,11 +297,7 @@ services: - awx volumes: - "loki_storage:/loki:rw" - #- "../../docker-compose/loki/volumes/index:/loki/index" - #- "../../docker-compose/loki/volumes/boltdb-cache:/loki/boltdb-cache" - "../../loki/local-config.yaml:/etc/loki/local-config.yaml" - depends_on: - - grafana {% endif %} {% if execution_node_count|int > 0 %} diff --git a/tools/otel/otel-collector-config.yaml b/tools/otel/otel-collector-config.yaml index ebbf0606cb..26a330cf5d 100644 --- a/tools/otel/otel-collector-config.yaml +++ b/tools/otel/otel-collector-config.yaml @@ -2,11 +2,22 @@ receivers: otlp: protocols: grpc: + http: exporters: debug: verbosity: detailed + file: + path: /awx-logs/awx-logs.json.zstd + rotation: + max_days: 14 + localtime: false + max_megabytes: 300 + max_backups: 200 + format: json + compression: zstd + loki: endpoint: http://loki:3100/loki/api/v1/push tls: @@ -32,7 +43,7 @@ service: logs: receivers: [otlp] processors: [batch] - exporters: [loki] + exporters: [file, loki] extensions: - health_check -- cgit v1.2.3