Add recording

* Always output awx logs to a file via otel
* That log file can always be later replayed into a product that
  supports otlp at a later date.
* Useful when you find a problem that you need a time series DB to help
  find and solve.
* Useful if a community member or customer has a problem where a time
  series db would be helpful. You can take a "remote" users log and
  replay it locally for analysis.

5 files changed, 45 insertions, 7 deletions

diff --git a/licenses/pyzstd.txt b/licenses/pyzstd.txt
new file mode 100644
index 0000000000..e69dae4dd4
--- /dev/null
+++ b/licenses/pyzstd.txt
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020-present, Ma Lin
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/requirements/requirements.in b/requirements/requirements.in
index fd1efc7734..5e25a766da 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -55,6 +55,7 @@ python-dsv-sdk>=1.0.4
 python-tss-sdk>=1.2.1
 python-ldap
 pyyaml>=6.0.1
+pyzstd
 receptorctl
 social-auth-core[openidconnect]==4.4.2  # see UPGRADE BLOCKERs
 social-auth-app-django==5.4.0  # see UPGRADE BLOCKERs
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index f1516afce5..502c4a1b52 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -446,6 +446,8 @@ pyyaml==6.0.1
     #   djangorestframework-yaml
     #   kubernetes
     #   receptorctl
+pyzstd==0.15.10
+    # via -r /awx_devel/requirements/requirements.in
 receptorctl==1.4.4
     # via -r /awx_devel/requirements/requirements.in
 redis==5.0.1
diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
index c6a0b4ed90..86f84523a3 100644
--- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
+++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
@@ -283,8 +283,7 @@ services:
       - "55679:55679" # zpages http://localhost:55679/debug/servicez /tracez
     volumes:
       - "../../otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml"
-    depends_on:
-      - loki
+      - "../../otel/awx-logs:/awx-logs/"
 {% endif %}
 {% if enable_loki|bool %}
   loki:
@@ -298,11 +297,7 @@ services:
       - awx
     volumes:
       - "loki_storage:/loki:rw"
-      #- "../../docker-compose/loki/volumes/index:/loki/index"
-      #- "../../docker-compose/loki/volumes/boltdb-cache:/loki/boltdb-cache"
       - "../../loki/local-config.yaml:/etc/loki/local-config.yaml"
-    depends_on:
-      - grafana
 {% endif %}
 
 {% if execution_node_count|int > 0 %}
diff --git a/tools/otel/otel-collector-config.yaml b/tools/otel/otel-collector-config.yaml
index ebbf0606cb..26a330cf5d 100644
--- a/tools/otel/otel-collector-config.yaml
+++ b/tools/otel/otel-collector-config.yaml
@@ -2,11 +2,22 @@ receivers:
   otlp:
     protocols:
       grpc:
+      http:
 
 exporters:
   debug:
     verbosity: detailed
 
+  file:
+    path: /awx-logs/awx-logs.json.zstd
+    rotation:
+      max_days: 14
+      localtime: false
+      max_megabytes: 300
+      max_backups: 200
+    format: json
+    compression: zstd
+
   loki:
     endpoint: http://loki:3100/loki/api/v1/push
     tls:
@@ -32,7 +43,7 @@ service:
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [loki]
+      exporters: [file, loki]
 
   extensions:
     - health_check