wip: otel stuff

environments/server/default.nix

@@ -32,6 +32,10 @@
      ";
   };
 in {
+  imports = [
+    ./otel.nix
+  ];
+
   # Make it use predictable interface names starting with eth0
   boot.kernelParams = ["net.ifnames=0"];
 
@@ -53,18 +57,6 @@ in {
     maxretry = 5;
   };
 
-  # virtualisation.oci-containers.containers."watchtower" = {
-  #   autoStart = true;
-  #   image = "containrrr/watchtower";
-  #   volumes = [
-  #     "/var/run/docker.sock:/var/run/docker.sock"
-  #   ];
-  #   environment = {
-  #     WATCHTOWER_CLEANUP = "true";
-  #     WATCHTOWER_POLL_INTERVAL = "86400";
-  #   };
-  # };
-
   networking.firewall.allowedTCPPorts = [80 443];
   services.nginx = {
     enable = true;
@@ -132,10 +124,27 @@ in {
       };
     };
 
+    virtualHosts."otel.sdg.moe" = {
+      enableACME = true;
+      addSSL = true;
+      locations."/" = {
+        proxyPass = "http://localhost:4318";
+        proxyWebsockets = true;
+        extraConfig = "proxy_pass_header Authorization;";
+      };
+    };
+    # virtualHosts."otel-groc.sdg.moe" = {
+    #   enableACME = true;
+    #   addSSL = true;
+    #   locations."/" = {
+    #     grpcPass = "http://localhost:4317";
+    #   };
+    # };
+
     virtualHosts."grafana.sdg.moe" = {
       enableACME = true;
       addSSL = true;
-      location."/" = {
+      locations."/" = {
         proxyPass = "http://localhost:1892";
         proxyWebsockets = true;
         recommendedProxySettings = true;
@@ -192,19 +201,4 @@ in {
       DISABLE_SSH = true;
     };
   };
-
-  services.opentelemetry-collector = {
-    enable = true;
-  };
-
-  services.grafana = {
-    enable = true;
-    settings = {
-      server = {
-        http_addr = "localhost";
-        http_port = 1892;
-        domain = "grafana.sdg.moe";
-      };
-    };
-  };
 }

environments/server/otel.nix

@@ -0,0 +1,68 @@
+{pkgs, ...}: {
+
+  services.opentelemetry-collector = {
+    enable = true;
+    package = pkgs.opentelemetry-collector-contrib;
+    configFile = ./otelcol.yaml;
+  };
+
+  services.mimir = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 1880;
+      };
+    };
+  };
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 1881;
+      };
+      storage_config = {
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+      common = {
+        path_prefix = "/var/lib/loki";
+      };
+      schema_config = {
+        configs = [
+          {
+            from = "2020-05-15";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema= "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+    };
+  };
+
+  services.tempo = {
+    enable = true;
+    settings = {
+      server = {
+        http_listen_port = 1882;
+      };
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 1892;
+        domain = "grafana.sdg.moe";
+      };
+    };
+  };
+}

environments/server/otelcol.yaml

@@ -0,0 +1,45 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+        auth:
+          authenticator: basicauth/server
+      http:
+        endpoint: localhost:4318
+        auth:
+          authenticator: basicauth/server
+
+extensions:
+  basicauth/server:
+    htpasswd:
+      inline: |
+        otel-user:thisissecretpleasedontuseit
+
+processors:
+  batch:
+
+exporters:
+  otlp/jaeger: # Jaeger supports OTLP directly
+    endpoint: https://jaeger.example.com:4317
+  prometheusremotewrite: # the PRW exporter, to ingest metrics to backend
+    endpoint: https://prw.example.com/v1/api/remote_write
+  file: # the File Exporter, to ingest logs to local file
+    path: ./app42_example.log
+    rotation:
+
+service:
+  extensions: [basicauth/server]
+  pipelines:
+    traces/dev:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/jaeger]
+    metrics/prod:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheusremotewrite]
+    logs/dev:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file]

modules/cli/nix/impermanence.nix

@@ -23,6 +23,7 @@
       "/var/lib/flatpak"
       "/var/lib/tcc"
       "/var/lib/flood"
+      "/var/lib/loki"
       "/var/lib/transmission"
       "/var/lib/gitea"
       "/var/lib/acme"