{ config, pkgs, lib, ... }: let mainCfg = config.nasp; nvidiaCfg = mainCfg.nvidia; dockerCfg = mainCfg.docker; nginxCfg = mainCfg.nginx; registryCfg = mainCfg.registry; sopsCfg = mainCfg.sops; telegrafCfg = mainCfg.telegraf; gCfg = mainCfg.gSeries; dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew)); in { options.nasp = { enable = lib.mkEnableOption "the nasp server configurations"; nvidia = { enable = lib.mkEnableOption "nvidia gpu support"; }; docker = { enable = lib.mkEnableOption "docker runtime"; }; nginx = { enable = lib.mkEnableOption "nginx web server"; enableCodeServer = lib.mkEnableOption "proxy code server in docker"; }; registry = { enable = lib.mkEnableOption "the nasp registry"; }; sops = { enable = lib.mkEnableOption "sops"; }; telegraf = { enable = lib.mkEnableOption "telegraf"; bucket = lib.mkOption { type = lib.types.str; description = "target influxdb bucket"; default = "trash"; }; }; gSeries = { enable = lib.mkEnableOption "the g-series server configurations"; serial = lib.mkOption { type = lib.types.int; description = "Serial of the machine (gX)"; }; eth0Name = lib.mkOption { type = lib.types.str; description = "Name of eth0 (192.168.16.0/24)"; }; eth1Name = lib.mkOption { type = lib.types.str; description = "Name of eth1 (thunet)"; }; eth2Name = lib.mkOption { type = lib.types.str; description = "Name of the RoCE NIC"; }; }; }; # inplementation config = lib.mkIf mainCfg.enable (lib.mkMerge [ # base { ## nix nix.settings.experimental-features = [ "nix-command" "flakes" ]; nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ]; ## hardware and system boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; time.hardwareClockInLocalTime = true; i18n.defaultLocale = "C.UTF-8"; i18n.extraLocaleSettings = lib.mkDefault { LC_ADDRESS = "zh_CN.UTF-8"; LC_IDENTIFICATION = "zh_CN.UTF-8"; LC_MEASUREMENT = "zh_CN.UTF-8"; LC_MONETARY = "zh_CN.UTF-8"; LC_NAME = "zh_CN.UTF-8"; LC_NUMERIC = "zh_CN.UTF-8"; LC_PAPER = "zh_CN.UTF-8"; LC_TELEPHONE = "zh_CN.UTF-8"; LC_TIME = "zh_CN.UTF-8"; }; time.timeZone = lib.mkDefault "Asia/Shanghai"; ## network services.resolved.enable = true; networking.networkmanager.enable = false; networking.useDHCP = false; systemd.network.enable = true; networking.firewall.allowedTCPPorts = [ 12022 ]; ## packages and services nixpkgs.config.allowUnfree = true; environment.systemPackages = with pkgs; [ bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh # extended acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat sysstat tcpdump unzip usbutils virt-what zip # full wireshark zmap ]; programs.zsh.enable = true; services.cron.enable = true; services.openssh.enable = true; services.openssh.settings.PermitRootLogin = "prohibit-password"; services.openssh.settings.PasswordAuthentication = false; services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ]; services.openssh.ports = [ 12022 ]; systemd.targets.sleep.enable = false; systemd.targets.suspend.enable = false; systemd.targets.hibernate.enable = false; systemd.targets.hybrid-sleep.enable = false; ## users users.mutableUsers = true; users.users.root.openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhS4voo3K/Dvzqckr0bouO1WkCI5XxswstHWnuuyKBz ltp1-bd" ]; users.users.root.shell = pkgs.zsh; system.activationScripts.dotfilesSetup.text = '' if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then source ${config.system.build.setEnvironment} rm -rf ~/dotfiles bash <(curl -fsSL dotfiles.cn) fi ''; users.users.nasp = { isNormalUser = true; createHome = true; group = "nasp"; extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user. packages = with pkgs; [ firefox ]; hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1"; }; users.groups.nasp = {}; ## desktop services.xserver = { enable = true; displayManager.gdm.enable = true; desktopManager.gnome.enable = true; xkb.layout = "us"; }; } (lib.mkIf (config.networking.hostId != null) { boot.supportedFilesystems = [ "zfs" ]; boot.zfs.forceImportRoot = false; services.zfs.autoScrub.enable = true; }) # nvidia (lib.mkIf (nvidiaCfg.enable) { nixpkgs.config.nvidia.acceptLicense = true; hardware.nvidia = { package = config.boot.kernelPackages.nvidiaPackages.legacy_470; modesetting.enable = false; powerManagement.enable = false; powerManagement.finegrained = false; open = false; nvidiaSettings = true; }; hardware.opengl = { enable = true; driSupport = true; driSupport32Bit = true; }; hardware.nvidia-container-toolkit.enable = true; systemd.services.nvidia-container-toolkit-cdi-generator = { path = [ pkgs.jq pkgs.moreutils ]; postStart = '' jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json ''; }; services.xserver.videoDrivers = [ "nvidia" ]; }) # docker (lib.mkIf (dockerCfg.enable) { virtualisation.docker = { enable = true; package = pkgs.docker_25; daemon.settings = { ipv6 = true; fixed-cidr-v6 = "fddd:d0c1:1::/64"; experimental = true; ip6tables = true; live-restore = false; }; }; environment.systemPackages = with pkgs; [ dnew ]; }) # nginx (lib.mkIf (nginxCfg.enable) { services.nginx = { enable = true; virtualHosts."default" = { serverName = "_"; default = true; locations."/" = { return = "404"; }; }; }; }) (lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) { services.nginx.virtualHosts."code-server" = { serverName = "proxy.nasp.fit"; locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = { extraConfig = '' rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break; proxy_pass "http://unix:/home2/run/$1.sock"; proxy_set_header Host $host; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection upgrade; proxy_set_header Accept-Encoding gzip; ''; }; }; }) # registry (lib.mkIf (registryCfg.enable) { systemd.timers."registry" = { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = "*:0/5:0"; Unit = "registry.service"; }; }; systemd.services."registry" = { serviceConfig = { Type = "oneshot"; User = "root"; }; script = builtins.readFile ./scripts/registry.sh; path = with pkgs; [ git bash su shadow getent ]; }; security.sudo.extraConfig = '' %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/docker %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/reboot %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/whoami %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/nvidia-smi %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/shutdown %nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/ip ''; }) # sops-nix ## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8" ## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age' ## mkdir -p ~/.config/sops/age ## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt" (lib.mkIf sopsCfg.enable { sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml; sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; }) # telegraf (lib.mkIf (telegrafCfg.enable) { sops.secrets.telegraf = assert sopsCfg.enable; {}; services.telegraf = { enable = true; environmentFiles = [ "/run/secrets/telegraf" ]; extraConfig = { agent = { interval = "3s"; round_interval = true; metric_batch_size = 1000; metric_buffer_limit = 10000; collection_jitter = "0s"; flush_interval = "30s"; flush_jitter = "3s"; precision = "0s"; hostname = assert (config.networking.hostName != ""); config.networking.hostName; omit_hostname = false; }; outputs.influxdb_v2 = { urls = [ "\${INFLUX_URL}" ]; token = "\${INFLUX_TOKEN}"; organization = "nasp.fit"; bucket = "trash"; bucket_tag = "bucket"; exclude_bucket_tag = true; timeout = "5s"; }; inputs.system = { name_override = "system"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.cpu = { percpu = false; name_override = "cpu"; tags = { bucket = telegrafCfg.bucket; }; }; inputs.kernel = { name_override = "kernel"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.processes = { name_override = "processes"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.mem = { name_override = "memory"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.disk = { ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"]; name_override = "disk"; tags = { bucket = telegrafCfg.bucket; }; interval = "30s"; }; inputs.diskio = { name_override = "diskio"; tags = { bucket = telegrafCfg.bucket; }; }; inputs.net = { name_override = "net"; ignore_protocol_stats = true; tags = { bucket = telegrafCfg.bucket; }; }; inputs.smart = { path_smartctl = "/run/current-system/sw/bin/smartctl"; use_sudo = true; name_override = "smartctl"; tags = { bucket = telegrafCfg.bucket; }; interval = "30s"; }; inputs.temp = { name_override = "temperture"; tags = { bucket = telegrafCfg.bucket; }; }; }; }; security.sudo.extraConfig = '' %telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl ''; }) # g series (lib.mkIf (gCfg.enable) (let ipSuffix = (builtins.toString (gCfg.serial + 100)); in { ## network networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial); networking.search = [ "nasp" ]; networking.nameservers = [ "192.168.16.118" ]; systemd.network.networks."10-eth0" = { matchConfig.Name = gCfg.eth0Name; networkConfig = { DHCP = "no"; IPv6AcceptRA = true; }; address = [ "192.168.16.${ipSuffix}/24" ]; routes = [ { routeConfig = { Gateway = "192.168.16.118"; GatewayOnLink = true; Metric = 90; }; } { routeConfig = { Gateway = "2001:da8:bf:300::1"; GatewayOnLink = true; Metric = 90; }; } ]; }; systemd.network.networks."10-eth1" = { matchConfig.Name = gCfg.eth1Name; networkConfig = { DHCP = "yes"; IPv6AcceptRA = true; }; linkConfig.RequiredForOnline = "no"; }; systemd.network.networks."10-eth2" = { matchConfig.Name = gCfg.eth2Name; address = [ "12.12.12.${ipSuffix}/24" ]; linkConfig.RequiredForOnline = "no"; }; networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true; networking.extraHosts = '' 192.168.16.101 g1-nasp g1 192.168.16.102 g2-nasp g2 192.168.16.103 g3-nasp g3 192.168.16.104 g4-nasp g4 192.168.16.105 g5-nasp g5 192.168.16.106 g6-nasp g6 192.168.16.107 g7-nasp g7 192.168.16.108 g8-nasp g8 192.168.16.109 g9-nasp g9 192.168.16.110 g10-nasp g10 192.168.16.111 g11-nasp g11 192.168.16.112 g12-nasp g12 192.168.16.113 g13-nasp g13 192.168.16.114 g14-nasp g14 192.168.16.115 g15-nasp g15 192.168.16.116 g16-nasp g16 192.168.16.117 g17-nasp g17 192.168.16.118 g18-nasp g18 nasp.fit git.nasp.fit 192.168.16.119 g19-nasp g19 192.168.16.120 g20-nasp g20 ''; networking.firewall.extraCommands = '' iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT ''; networking.rxe = { enable = true; interfaces = [ "${gCfg.eth2Name}" ]; }; fileSystems."/share" = { device = "192.168.16.118:/data0/share"; fsType = "nfs"; }; ## packages and services nasp.docker.enable = lib.mkDefault true; nasp.nvidia.enable = lib.mkDefault true; nasp.registry.enable = lib.mkDefault true; nasp.nginx.enable = lib.mkDefault true; nasp.nginx.enableCodeServer = lib.mkDefault true; nasp.sops.enable = lib.mkDefault true; nasp.telegraf = { enable = lib.mkDefault true; bucket = "g-series"; }; services.telegraf.extraConfig = { inputs.nvidia_smi = { bin_path = "/run/current-system/sw/bin/nvidia-smi"; tags = { bucket = "g-series"; }; }; }; services.ntp = { enable = true; servers = [ "192.168.16.118" ]; }; })) ]); }