{ config, pkgs, lib, ... }: let mainCfg = config.nasp; nvidiaCfg = mainCfg.nvidia; dockerCfg = mainCfg.docker; nginxCfg = mainCfg.nginx; registryCfg = mainCfg.registry; sopsCfg = mainCfg.sops; telegrafCfg = mainCfg.telegraf; gCfg = mainCfg.gSeries; dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew)); in { options.nasp = { enable = lib.mkEnableOption "the nasp server configurations"; nvidia = { enable = lib.mkEnableOption "nvidia gpu support"; }; docker = { enable = lib.mkEnableOption "docker runtime"; }; nginx = { enable = lib.mkEnableOption "nginx web server"; enableCodeServer = lib.mkEnableOption "proxy code server in docker"; }; registry = { enable = lib.mkEnableOption "the nasp registry"; }; sops = { enable = lib.mkEnableOption "sops"; }; telegraf = { enable = lib.mkEnableOption "telegraf"; bucket = lib.mkOption { type = lib.types.str; description = "target influxdb bucket"; default = "trash"; }; }; gSeries = { enable = lib.mkEnableOption "the g-series server configurations"; serial = lib.mkOption { type = lib.types.int; description = "Serial of the machine (gX)"; }; eth0Name = lib.mkOption { type = lib.types.str; default = ""; description = "Name of eth0 (192.168.16.0/24)"; }; eth1Name = lib.mkOption { type = lib.types.str; default = ""; description = "Name of eth1 (thunet)"; }; eth2Name = lib.mkOption { type = lib.types.str; default = ""; description = "Name of the RoCE NIC"; }; }; }; # inplementation config = lib.mkIf mainCfg.enable (lib.mkMerge [ # base { ## nix nix.settings.experimental-features = [ "nix-command" "flakes" ]; nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ]; ## hardware and system boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; time.hardwareClockInLocalTime = true; i18n.defaultLocale = "C.UTF-8"; i18n.extraLocaleSettings = lib.mkDefault { LC_ADDRESS = "zh_CN.UTF-8"; LC_IDENTIFICATION = "zh_CN.UTF-8"; LC_MEASUREMENT = "zh_CN.UTF-8"; LC_MONETARY = "zh_CN.UTF-8"; LC_NAME = "zh_CN.UTF-8"; LC_NUMERIC = "zh_CN.UTF-8"; LC_PAPER = "zh_CN.UTF-8"; LC_TELEPHONE = "zh_CN.UTF-8"; LC_TIME = "zh_CN.UTF-8"; }; time.timeZone = lib.mkDefault "Asia/Shanghai"; ## network services.resolved.enable = true; networking.networkmanager.enable = false; networking.useDHCP = false; systemd.network.enable = true; networking.firewall.allowedTCPPorts = [ 12022 ]; ## packages and services nixpkgs.config.allowUnfree = true; environment.systemPackages = with pkgs; [ bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh # extended acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat sysstat tcpdump unzip usbutils virt-what zip # full wireshark zmap ]; programs.zsh.enable = true; programs.nix-ld.enable = true; services.cron.enable = true; services.openssh.enable = true; services.openssh.settings.PermitRootLogin = "prohibit-password"; services.openssh.settings.PasswordAuthentication = false; services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ]; services.openssh.ports = [ 12022 ]; systemd.targets.sleep.enable = false; systemd.targets.suspend.enable = false; systemd.targets.hibernate.enable = false; systemd.targets.hybrid-sleep.enable = false; ## users users.mutableUsers = true; users.users.root.openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyZILj+GxTUhdCgz2w1TxQ+aTcggnOJIb84qA4u271S asz258-17ac-bm-v0" ]; users.users.root.shell = pkgs.zsh; system.activationScripts.dotfilesSetup.text = '' if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then source ${config.system.build.setEnvironment} rm -rf ~/dotfiles bash <(curl -fsSL dotfiles.cn) fi ''; users.users.nasp = { isNormalUser = true; createHome = true; group = "nasp"; extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user. packages = with pkgs; [ firefox ]; hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1"; }; users.groups.nasp = {}; ## desktop services.xserver = { enable = true; displayManager.gdm.enable = true; desktopManager.gnome.enable = true; xkb.layout = "us"; }; } (lib.mkIf (config.networking.hostId != null) { boot.supportedFilesystems = [ "zfs" ]; boot.zfs.forceImportRoot = false; services.zfs.autoScrub.enable = true; }) # nvidia (lib.mkIf (nvidiaCfg.enable) { nixpkgs.config.nvidia.acceptLicense = true; hardware.nvidia = { package = config.boot.kernelPackages.nvidiaPackages.legacy_470; modesetting.enable = false; powerManagement.enable = false; powerManagement.finegrained = false; open = false; nvidiaSettings = true; }; hardware.graphics = { enable = true; enable32Bit = true; }; hardware.nvidia-container-toolkit.enable = true; systemd.services.nvidia-container-toolkit-cdi-generator = { path = [ pkgs.jq pkgs.moreutils ]; postStart = '' jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json ''; }; services.xserver.videoDrivers = [ "nvidia" ]; }) # docker (lib.mkIf (dockerCfg.enable) { virtualisation.docker = { enable = true; package = pkgs.docker_25; daemon.settings = { ipv6 = true; fixed-cidr-v6 = "fddd:d0c1:1::/64"; experimental = true; ip6tables = true; live-restore = false; }; }; environment.systemPackages = with pkgs; [ dnew ]; }) # nginx (lib.mkIf (nginxCfg.enable) { services.nginx = { enable = true; virtualHosts."default" = { serverName = "_"; default = true; locations."/" = { return = "404"; }; }; }; }) (lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) { services.nginx.virtualHosts."code-server" = { serverName = "proxy.nasp.fit"; locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = { extraConfig = '' rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break; proxy_pass "http://unix:/home2/run/$1.sock"; proxy_set_header Host $host; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection upgrade; proxy_set_header Accept-Encoding gzip; ''; }; }; }) # registry (lib.mkIf (registryCfg.enable) { systemd.timers."registry" = { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = "*:0/5:0"; Unit = "registry.service"; }; }; systemd.services."registry" = { serviceConfig = { Type = "oneshot"; User = "root"; }; script = builtins.readFile ./scripts/registry.sh; path = with pkgs; [ git bash su shadow getent ]; }; security.sudo.extraConfig = '' %nasp ALL = (root) NOPASSWD: ALL ''; }) # sops-nix ## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8" ## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age' ## mkdir -p ~/.config/sops/age ## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt" (lib.mkIf sopsCfg.enable { sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml; sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; }) # telegraf (lib.mkIf (telegrafCfg.enable) { sops.secrets.telegraf = assert sopsCfg.enable; {}; services.telegraf = { enable = true; environmentFiles = [ "/run/secrets/telegraf" ]; extraConfig = { agent = { interval = "3s"; round_interval = true; metric_batch_size = 1000; metric_buffer_limit = 10000; collection_jitter = "0s"; flush_interval = "30s"; flush_jitter = "3s"; precision = "0s"; hostname = assert (config.networking.hostName != ""); config.networking.hostName; omit_hostname = false; }; outputs.influxdb_v2 = { urls = [ "\${INFLUX_URL}" ]; token = "\${INFLUX_TOKEN}"; organization = "nasp.fit"; bucket = "trash"; bucket_tag = "bucket"; exclude_bucket_tag = true; timeout = "5s"; }; inputs.system = { name_override = "system"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.cpu = { percpu = false; name_override = "cpu"; tags = { bucket = telegrafCfg.bucket; }; }; inputs.kernel = { name_override = "kernel"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.processes = { name_override = "processes"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.mem = { name_override = "memory"; tags = { bucket = telegrafCfg.bucket; }; interval = "10s"; }; inputs.disk = { ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"]; name_override = "disk"; tags = { bucket = telegrafCfg.bucket; }; interval = "30s"; }; inputs.diskio = { name_override = "diskio"; tags = { bucket = telegrafCfg.bucket; }; }; inputs.net = { name_override = "net"; ignore_protocol_stats = true; tags = { bucket = telegrafCfg.bucket; }; }; inputs.smart = { path_smartctl = "/run/current-system/sw/bin/smartctl"; use_sudo = true; name_override = "smartctl"; tags = { bucket = telegrafCfg.bucket; }; interval = "30s"; }; inputs.temp = { name_override = "temperture"; tags = { bucket = telegrafCfg.bucket; }; }; }; }; security.sudo.extraConfig = '' %telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl ''; }) (lib.mkIf (telegrafCfg.enable && nvidiaCfg.enable) { services.telegraf.extraConfig = { inputs.nvidia_smi = { bin_path = "/run/current-system/sw/bin/nvidia-smi"; tags = { bucket = telegrafCfg.bucket; }; }; }; }) # g series (lib.mkIf (gCfg.enable) { ## network networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial); networking.search = [ "" ]; networking.nameservers = [ "192.168.16.1" ]; networking.extraHosts = '' 192.168.16.1 nasp.fit git.nasp.fit 192.168.16.101 g1.nasp g1 192.168.16.102 g2.nasp g2 192.168.16.103 g3.nasp g3 192.168.16.104 g4.nasp g4 192.168.16.105 g5.nasp g5 192.168.16.106 g6.nasp g6 192.168.16.107 g7.nasp g7 192.168.16.108 g8.nasp g8 192.168.16.109 g9.nasp g9 192.168.16.110 g10.nasp g10 192.168.16.111 g11.nasp g11 192.168.16.112 g12.nasp g12 192.168.16.113 g13.nasp g13 192.168.16.114 g14.nasp g14 192.168.16.115 g15.nasp g15 192.168.16.116 g16.nasp g16 192.168.16.117 g17.nasp g17 192.168.16.118 g18.nasp g18 192.168.16.119 g19.nasp g19 ''; networking.firewall.extraCommands = '' iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT ''; fileSystems."/gshare" = { device = "192.168.16.1:/data1/share"; fsType = "nfs"; }; ## packages and services nasp.docker.enable = lib.mkDefault true; nasp.nvidia.enable = lib.mkDefault true; nasp.registry.enable = lib.mkDefault true; nasp.nginx.enable = lib.mkDefault true; nasp.nginx.enableCodeServer = lib.mkDefault true; nasp.sops.enable = lib.mkDefault true; nasp.telegraf = { enable = lib.mkDefault true; bucket = "g-series"; }; services.ntp = { enable = true; servers = [ "192.168.16.1" ]; }; }) (lib.mkIf (gCfg.enable && gCfg.eth0Name != "") { systemd.network.networks."10-eth0" = { matchConfig.Name = gCfg.eth0Name; networkConfig = { DHCP = "no"; IPv6AcceptRA = true; }; address = [ "192.168.16.${builtins.toString (gCfg.serial + 100)}/24" ]; routes = [ { Gateway = "192.168.16.1"; GatewayOnLink = true; Metric = 90; } { Gateway = "fd01:da8:bf:300::1"; GatewayOnLink = true; Metric = 90; } ]; }; networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true; }) (lib.mkIf (gCfg.enable && gCfg.eth1Name != "") { systemd.network.networks."10-eth1" = { matchConfig.Name = gCfg.eth1Name; networkConfig = { DHCP = "yes"; IPv6AcceptRA = true; }; linkConfig.RequiredForOnline = "no"; }; }) (lib.mkIf (gCfg.enable && gCfg.eth2Name != "") { systemd.network.networks."10-eth2" = { matchConfig.Name = gCfg.eth2Name; address = [ "12.12.12.${builtins.toString (gCfg.serial + 100)}/24" ]; linkConfig.RequiredForOnline = "no"; }; networking.rxe = { enable = true; interfaces = [ "${gCfg.eth2Name}" ]; }; }) ]); }