NixOS-Config/hosts/modules/nasp.nix

{ config, pkgs, lib, ... }:

let
  mainCfg = config.nasp;
  nvidiaCfg = mainCfg.nvidia;
  dockerCfg = mainCfg.docker;
  nginxCfg = mainCfg.nginx;
  registryCfg = mainCfg.registry;
  sopsCfg = mainCfg.sops;
  telegrafCfg = mainCfg.telegraf;
  gCfg = mainCfg.gSeries;
  dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
in
{
  options.nasp = {
    enable = lib.mkEnableOption "the nasp server configurations";
    nvidia = {
      enable = lib.mkEnableOption "nvidia gpu support";
    };
    docker = {
      enable = lib.mkEnableOption "docker runtime";
    };
    nginx = {
      enable = lib.mkEnableOption "nginx web server";
      enableCodeServer = lib.mkEnableOption "proxy code server in docker";
    };
    registry = {
      enable = lib.mkEnableOption "the nasp registry";
    };
    sops = {
      enable = lib.mkEnableOption "sops";
    };
    telegraf = {
      enable = lib.mkEnableOption "telegraf";
      bucket = lib.mkOption {
        type = lib.types.str;
        description = "target influxdb bucket";
        default = "trash";
      };
    };
    gSeries = {
      enable = lib.mkEnableOption "the g-series server configurations";
      serial = lib.mkOption {
        type = lib.types.int;
        description = "Serial of the machine (gX)";
      };
      eth0Name = lib.mkOption {
        type = lib.types.str;
        description = "Name of eth0 (192.168.16.0/24)";
      };
      eth1Name = lib.mkOption {
        type = lib.types.str;
        description = "Name of eth1 (thunet)";
      };
      eth2Name = lib.mkOption {
        type = lib.types.str;
        description = "Name of the RoCE NIC";
      };
    };
  };

  # inplementation
  config = lib.mkIf mainCfg.enable (lib.mkMerge [
    # base
    {
      ## nix
      nix.settings.experimental-features = [ "nix-command" "flakes" ];
      nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
      ## hardware and system
      boot.loader.systemd-boot.enable = true;
      boot.loader.efi.canTouchEfiVariables = true;
      time.hardwareClockInLocalTime = true;
      i18n.defaultLocale = "C.UTF-8";
      i18n.extraLocaleSettings = lib.mkDefault {
        LC_ADDRESS = "zh_CN.UTF-8";
        LC_IDENTIFICATION = "zh_CN.UTF-8";
        LC_MEASUREMENT = "zh_CN.UTF-8";
        LC_MONETARY = "zh_CN.UTF-8";
        LC_NAME = "zh_CN.UTF-8";
        LC_NUMERIC = "zh_CN.UTF-8";
        LC_PAPER = "zh_CN.UTF-8";
        LC_TELEPHONE = "zh_CN.UTF-8";
        LC_TIME = "zh_CN.UTF-8";
      };
      time.timeZone = lib.mkDefault "Asia/Shanghai";
      ## network
      services.resolved.enable = true;
      networking.networkmanager.enable = false;
      networking.useDHCP = false;
      systemd.network.enable = true;
      networking.firewall.allowedTCPPorts = [ 12022 ];
      ## packages and services
      nixpkgs.config.allowUnfree = true;
      environment.systemPackages = with pkgs; [
        bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man
        openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh
        # extended
        acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw
        lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat
        sysstat tcpdump unzip usbutils virt-what zip
        # full
        wireshark zmap
      ];
      programs.zsh.enable = true;
      services.cron.enable = true;
      services.openssh.enable = true;
      services.openssh.settings.PermitRootLogin = "prohibit-password";
      services.openssh.settings.PasswordAuthentication = false;
      services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ];
      services.openssh.ports = [ 12022 ];
      systemd.targets.sleep.enable = false;
      systemd.targets.suspend.enable = false;
      systemd.targets.hibernate.enable = false;
      systemd.targets.hybrid-sleep.enable = false;
      ## users
      users.mutableUsers = true;
      users.users.root.openssh.authorizedKeys.keys = [
        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a"
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows"
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhS4voo3K/Dvzqckr0bouO1WkCI5XxswstHWnuuyKBz ltp1-bd"
      ];
      users.users.root.shell = pkgs.zsh;
      system.activationScripts.dotfilesSetup.text = ''
        if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then
          source ${config.system.build.setEnvironment}
          rm -rf ~/dotfiles
          bash <(curl -fsSL dotfiles.cn)
        fi
      '';
      users.users.nasp = {
        isNormalUser = true;
        createHome = true;
        group = "nasp";
        extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
        packages = with pkgs; [
          firefox
        ];
        hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
      };
      users.groups.nasp = {};
      ## desktop
      services.xserver = {
        enable = true;
        displayManager.gdm.enable = true;
        desktopManager.gnome.enable = true;
        xkb.layout = "us";
      };
    }
    (lib.mkIf (config.networking.hostId != null) {
      boot.supportedFilesystems = [ "zfs" ];
      boot.zfs.forceImportRoot = false;
      services.zfs.autoScrub.enable = true;
    })
    # nvidia
    (lib.mkIf (nvidiaCfg.enable) {
      nixpkgs.config.nvidia.acceptLicense = true;
      hardware.nvidia = {
        package = config.boot.kernelPackages.nvidiaPackages.legacy_470;
        modesetting.enable = false;
        powerManagement.enable = false;
        powerManagement.finegrained = false;
        open = false;
        nvidiaSettings = true;
      };
      hardware.opengl = {
        enable = true;
        driSupport = true;
        driSupport32Bit = true;
      };
      hardware.nvidia-container-toolkit.enable = true;
      systemd.services.nvidia-container-toolkit-cdi-generator = {
        path = [ pkgs.jq pkgs.moreutils ];
        postStart = ''
         jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json
        '';
      };
      services.xserver.videoDrivers = [ "nvidia" ];
    })
    # docker
    (lib.mkIf (dockerCfg.enable) {
      virtualisation.docker = {
        enable = true;
        package = pkgs.docker_25;
        daemon.settings = {
          ipv6 = true;
          fixed-cidr-v6 = "fddd:d0c1:1::/64";
          experimental = true;
          ip6tables = true;
          live-restore = false;
        };
      };
      environment.systemPackages = with pkgs; [
        dnew
      ];
    })
    # nginx
    (lib.mkIf (nginxCfg.enable) {
      services.nginx = {
        enable = true;
        virtualHosts."default" = {
          serverName = "_";
          default = true;
          locations."/" = {
            return = "404";
          };
        };
      };
    })
    (lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) {
      services.nginx.virtualHosts."code-server" = {
        serverName = "proxy.nasp.fit";
        locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)"  = {
          extraConfig = ''
            rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
            proxy_pass "http://unix:/home2/run/$1.sock";
            proxy_set_header Host $host;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection upgrade;
            proxy_set_header Accept-Encoding gzip;
          '';
        };
      };
    })
    # registry
    (lib.mkIf (registryCfg.enable) {
      systemd.timers."registry" = {
        wantedBy = [ "timers.target" ];
        timerConfig = {
          OnCalendar = "*:0/5:0";
          Unit = "registry.service";
        };
      };
      systemd.services."registry" = {
        serviceConfig = {
          Type = "oneshot";
          User = "root";
        };
        script = builtins.readFile ./scripts/registry.sh;
        path = with pkgs; [ git bash su shadow getent ];
      };
      security.sudo.extraConfig = ''
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/docker
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/reboot
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/whoami
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/nvidia-smi
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/shutdown
        %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/ip
      '';
    })
    # sops-nix
    ## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8"
    ## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
    ## mkdir -p ~/.config/sops/age
    ## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt"
    (lib.mkIf sopsCfg.enable {
      sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml;
      sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
    })
    # telegraf
    (lib.mkIf (telegrafCfg.enable) {
      sops.secrets.telegraf = assert sopsCfg.enable; {};
      services.telegraf = {
        enable = true;
        environmentFiles = [ "/run/secrets/telegraf" ];
        extraConfig = {
          agent = {
            interval = "3s";
            round_interval = true;
            metric_batch_size = 1000;
            metric_buffer_limit = 10000;
            collection_jitter = "0s";
            flush_interval = "30s";
            flush_jitter = "3s";
            precision = "0s";
            hostname = assert (config.networking.hostName != ""); config.networking.hostName;
            omit_hostname = false;
          };
          outputs.influxdb_v2 = {
            urls = [ "\${INFLUX_URL}" ];
            token = "\${INFLUX_TOKEN}";
            organization = "nasp.fit";
            bucket = "trash";
            bucket_tag = "bucket";
            exclude_bucket_tag = true;
            timeout = "5s";
          };
          inputs.system = {
            name_override = "system";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "10s";
          };
          inputs.cpu = {
            percpu = false;
            name_override = "cpu";
            tags = { bucket = telegrafCfg.bucket; };
          };
          inputs.kernel = {
            name_override = "kernel";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "10s";
          };
          inputs.processes = {
            name_override = "processes";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "10s";
          };
          inputs.mem = {
            name_override = "memory";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "10s";
          };
          inputs.disk = {
            ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"];
            name_override = "disk";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "30s";
          };
          inputs.diskio = {
            name_override = "diskio";
            tags = { bucket = telegrafCfg.bucket; };
          };
          inputs.net = {
            name_override = "net";
            ignore_protocol_stats = true;
            tags = { bucket = telegrafCfg.bucket; };
          };
          inputs.smart = {
            path_smartctl = "/run/current-system/sw/bin/smartctl";
            use_sudo = true;
            name_override = "smartctl";
            tags = { bucket = telegrafCfg.bucket; };
            interval = "30s";
          };
          inputs.temp = {
            name_override = "temperture";
            tags = { bucket = telegrafCfg.bucket; };
          };
        };
      };
      security.sudo.extraConfig = ''
        %telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl
      '';
    })
    # g series
    (lib.mkIf (gCfg.enable) (let
      ipSuffix = (builtins.toString (gCfg.serial + 100)); in {
      ## network
      networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
      networking.search = [ "nasp" ];
      networking.nameservers = [ "192.168.16.118" ];
      systemd.network.networks."10-eth0" = {
        matchConfig.Name = gCfg.eth0Name;
        networkConfig = {
          DHCP = "no";
          IPv6AcceptRA = true;
        };
        address = [ "192.168.16.${ipSuffix}/24" ];
        routes = [
          {
            routeConfig = {
              Gateway = "192.168.16.118";
              GatewayOnLink = true;
              Metric = 90;
            };
          }
          {
            routeConfig = {
              Gateway = "fd01:da8:bf:300::1";
              GatewayOnLink = true;
              Metric = 90;
            };
          }
        ];
      };
      systemd.network.networks."10-eth1" = {
        matchConfig.Name = gCfg.eth1Name;
        networkConfig = {
          DHCP = "yes";
          IPv6AcceptRA = true;
        };
        linkConfig.RequiredForOnline = "no";
      };
      systemd.network.networks."10-eth2" = {
        matchConfig.Name = gCfg.eth2Name;
        address = [ "12.12.12.${ipSuffix}/24" ];
        linkConfig.RequiredForOnline = "no";
      };
      networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
      networking.extraHosts = ''
        192.168.16.101 g1-nasp g1
        192.168.16.102 g2-nasp g2
        192.168.16.103 g3-nasp g3
        192.168.16.104 g4-nasp g4
        192.168.16.105 g5-nasp g5
        192.168.16.106 g6-nasp g6
        192.168.16.107 g7-nasp g7
        192.168.16.108 g8-nasp g8
        192.168.16.109 g9-nasp g9
        192.168.16.110 g10-nasp g10
        192.168.16.111 g11-nasp g11
        192.168.16.112 g12-nasp g12
        192.168.16.113 g13-nasp g13
        192.168.16.114 g14-nasp g14
        192.168.16.115 g15-nasp g15
        192.168.16.116 g16-nasp g16
        192.168.16.117 g17-nasp g17
        192.168.16.118 g18-nasp g18 nasp.fit git.nasp.fit
        192.168.16.119 g19-nasp g19
        192.168.16.120 g20-nasp g20
      '';
      networking.firewall.extraCommands = ''
        iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT
        iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT
      '';
      networking.rxe = {
        enable = true;
        interfaces = [ "${gCfg.eth2Name}" ];
      };
      fileSystems."/share" = {
        device = "192.168.16.118:/data0/share";
        fsType = "nfs";
      };
      ## packages and services
      nasp.docker.enable = lib.mkDefault true;
      nasp.nvidia.enable = lib.mkDefault true;
      nasp.registry.enable = lib.mkDefault true;
      nasp.nginx.enable = lib.mkDefault true;
      nasp.nginx.enableCodeServer = lib.mkDefault true;
      nasp.sops.enable = lib.mkDefault true;
      nasp.telegraf = {
        enable = lib.mkDefault true;
        bucket = "g-series";
      };
      services.telegraf.extraConfig = {
        inputs.nvidia_smi = {
          bin_path = "/run/current-system/sw/bin/nvidia-smi";
          tags = { bucket = "g-series"; };
        };
      };
      services.ntp = {
        enable = true;
        servers = [ "192.168.16.118" ];
      };
    }))
  ]);
}