{
  pkgs,
  lib,
  config,
  instanceConfig,
  instanceConfigs,
  # inputs,
  ...
}:
with lib;
let
  cfg = config.xyno.services.monitoring;

  firstInstanceWithPromServer = (builtins.head (
    builtins.filter (x: x ? prometheusServer && x.prometheusServer) (attrValues instanceConfigs)
  )).hostName;
  vmBasicAuthUsername = "xyno-monitoring";
in
{
  options.xyno.services.monitoring.enable =
    mkEnableOption "enables monitoring (prometheus exporters and stuff)";
  options.xyno.services.monitoring.remoteWriteUrl = mkOption {
    type = types.str;
    default = "http://${firstInstanceWithPromServer}.${config.xyno.services.wireguard.monHostsDomain}:8428/api/v1/write";
    description = "where prometheus metrics should be pushed to";
  };
  options.xyno.services.monitoring.exporters = mkOption {
    type = types.attrsOf (types.either types.int types.str);
    description = "names of exporters and their ports (to open fw and generate prometheus config)";
    example = ''
      {
        node = 9100;
        postgres = "unix:///run/postgres-exporter.sock";
      }
    '';
  };

  config = mkMerge [
    (mkIf cfg.enable {
      services.prometheus.exporters.node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
      };
      xyno.services.monitoring.exporters.node = config.services.prometheus.exporters.node.port;
      services.vmagent = {
        remoteWrite.url = cfg.remoteWriteUrl;
        remoteWrite.basicAuthUsername = vmBasicAuthUsername;
        remoteWrite.basicAuthPasswordFile = config.sops.secrets."victoriametrics/basicAuthPassword".path;

        prometheusConfig.scrape_configs = mapAttrsToList (name: value: {
          job_name = "${name}-exporter";
          metrics_path = "/metrics";
          staticConfigs = [
            {
              targets = [ (if ((builtins.typeOf value) == "string") then value else "[::1]:${toString value}") ];
              labels.type = name;
              labels.host = config.networking.hostName;
            }
          ];
        }) cfg.exporters;
      };
      sops.secrets."victoriametrics/basicAuthPassword" = {
        reloadUnits = [ "vmagent.service" ];
      };

    })
    (mkIf (cfg.enable && instanceConfig ? prometheusServer && instanceConfig.prometheusServer) {
      xyno.impermanence.directories = [ "/var/lib/${config.services.victoriametrics.stateDir}" ];
      sops.secrets."victoriametrics/basicAuthPassword" = {
        reloadUnits = [ "victoriametrics.service" ];
      };
      networking.firewall.extraInputRules = ''tcp dport 8428 ip6 daddr ${config.xyno.services.wireguard.monIp6}/128 accept comment "victoriametrics-http"'';
      systemd.services.victoriametrics.serviceConfig.LoadCredential = [
        "basic_auth_pw:${config.sops.secrets."victoriametrics/basicAuthPassword".path}"
      ];
      services.victoriametrics = {
        enable = true;
        listenAddress = "${config.xyno.services.wireguard.monIp6}:8428";
        extraOptions = [
          "-httpAuth.username=${vmBasicAuthUsername}"
          "-httpAuth.password=file://\${CREDENTIALS_DIRECTORY}/basic_auth_pw"
        ];
      };

      services.grafana.declarativePlugins = with pkgs.grafanaPlugins; [ victoriametrics-metrics-datasource ];

    })
  ];
}