526 lines
17 KiB
Nix
526 lines
17 KiB
Nix
{ config, pkgs, lib, ... }:
|
||
|
||
let
|
||
mainCfg = config.nasp;
|
||
nvidiaCfg = mainCfg.nvidia;
|
||
dockerCfg = mainCfg.docker;
|
||
nginxCfg = mainCfg.nginx;
|
||
registryCfg = mainCfg.registry;
|
||
sopsCfg = mainCfg.sops;
|
||
telegrafCfg = mainCfg.telegraf;
|
||
gCfg = mainCfg.gSeries;
|
||
cCfg = mainCfg.cSeries;
|
||
dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
|
||
decToHex =
|
||
let
|
||
intToHex = [
|
||
"0" "1" "2" "3" "4" "5" "6" "7" "8" "9"
|
||
"a" "b" "c" "d" "e" "f"
|
||
];
|
||
toHex' = q: a:
|
||
if q > 0
|
||
then (toHex'
|
||
(q / 16)
|
||
((lib.elemAt intToHex (lib.mod q 16)) + a))
|
||
else a;
|
||
in
|
||
v: toHex' v "";
|
||
in
|
||
{
|
||
options.nasp = {
|
||
enable = lib.mkEnableOption "the nasp server configurations";
|
||
nvidia = {
|
||
enable = lib.mkEnableOption "nvidia gpu support";
|
||
};
|
||
docker = {
|
||
enable = lib.mkEnableOption "docker runtime";
|
||
};
|
||
nginx = {
|
||
enable = lib.mkEnableOption "nginx web server";
|
||
};
|
||
registry = {
|
||
enable = lib.mkEnableOption "the nasp registry";
|
||
};
|
||
sops = {
|
||
enable = lib.mkEnableOption "sops";
|
||
};
|
||
telegraf = {
|
||
enable = lib.mkEnableOption "telegraf";
|
||
bucket = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = "target influxdb bucket";
|
||
default = "trash";
|
||
};
|
||
};
|
||
gSeries = {
|
||
enable = lib.mkEnableOption "the g-series server configurations";
|
||
serial = lib.mkOption {
|
||
type = lib.types.ints.u8;
|
||
description = "Serial of the machine (gX)";
|
||
};
|
||
eth0Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of eth0 (192.168.16.0/24)";
|
||
};
|
||
eth1Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of eth1 (thunet)";
|
||
};
|
||
eth2Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of the RoCE NIC";
|
||
};
|
||
};
|
||
cSeries = {
|
||
enable = lib.mkEnableOption "cpu server configurations";
|
||
serial = lib.mkOption {
|
||
type = lib.types.ints.u8;
|
||
description = "Serial of the machine (cX)";
|
||
};
|
||
ethLanName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of the LAN NIC";
|
||
};
|
||
ethRDMAName = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of the RDMA NIC";
|
||
};
|
||
};
|
||
};
|
||
|
||
# inplementation
|
||
config = lib.mkIf mainCfg.enable (lib.mkMerge [
|
||
# base
|
||
{
|
||
## nix
|
||
nix = {
|
||
settings.experimental-features = [ "nix-command" "flakes" ];
|
||
settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
|
||
optimise.automatic = true;
|
||
};
|
||
## hardware and system
|
||
boot.loader.systemd-boot.enable = true;
|
||
boot.loader.efi.canTouchEfiVariables = true;
|
||
i18n.defaultLocale = "C.UTF-8";
|
||
i18n.extraLocaleSettings = lib.mkDefault {
|
||
LC_ADDRESS = "zh_CN.UTF-8";
|
||
LC_IDENTIFICATION = "zh_CN.UTF-8";
|
||
LC_MEASUREMENT = "zh_CN.UTF-8";
|
||
LC_MONETARY = "zh_CN.UTF-8";
|
||
LC_NAME = "zh_CN.UTF-8";
|
||
LC_NUMERIC = "zh_CN.UTF-8";
|
||
LC_PAPER = "zh_CN.UTF-8";
|
||
LC_TELEPHONE = "zh_CN.UTF-8";
|
||
LC_TIME = "zh_CN.UTF-8";
|
||
};
|
||
time.timeZone = lib.mkDefault "Asia/Shanghai";
|
||
## network
|
||
services.avahi.enable = true;
|
||
boot.kernel.sysctl = {
|
||
"net.core.default_qdisc" = "fq";
|
||
"net.ipv4.tcp_congestion_control" = "bbr";
|
||
};
|
||
services.resolved.enable = true;
|
||
networking.networkmanager.enable = false;
|
||
networking.useDHCP = false;
|
||
systemd.network.enable = true;
|
||
networking.nftables.enable = true;
|
||
networking.firewall.allowedTCPPorts = [ 12022 ];
|
||
networking.firewall.extraInputRules = ''
|
||
ip saddr 192.168.16.0/21 accept
|
||
ip6 saddr fd01:da8:bf::/48 accept
|
||
'';
|
||
networking.extraHosts = ''
|
||
192.168.16.1 ssh.nasp.fit git.nasp.fit jump.nasp.fit
|
||
192.168.16.115 g15.nasp g15 lm1
|
||
192.168.16.116 g16.nasp g16 lm2
|
||
192.168.20.101 c1 c1.nasp
|
||
192.168.20.102 c2 c2.nasp
|
||
'';
|
||
## packages and services
|
||
nixpkgs.config.allowUnfree = true;
|
||
environment.systemPackages = with pkgs; [
|
||
bash cmake curl dialog dig dmidecode e2fsprogs ethtool expect fd file fzf gcc git gnumake
|
||
gnupg htop iftop inetutils iotop iproute2 iputils jq less lrzsz lshw lsof man mtr
|
||
nettools nmap pciutils openssh openssl p7zip python3 ripgrep socat sops sudo tcpdump tmux
|
||
unzip usbutils util-linux vim wget zip zsh zssh zstd
|
||
acpi asciinema atop bat bridge-utils btop conntrack-tools dos2unix ffmpeg fish git-lfs
|
||
imagemagick iptstate killall libwebp ndisc6 netcat-gnu nethogs nix-diff nvme-cli pstree
|
||
pwgen smartmontools sysstat tldr virt-what
|
||
wireshark zmap
|
||
];
|
||
programs.zsh.enable = true;
|
||
programs.nix-ld.enable = true;
|
||
programs.git.lfs.enable = true;
|
||
services.cron.enable = true;
|
||
services.locate = {
|
||
enable = true;
|
||
package = pkgs.plocate;
|
||
};
|
||
fonts = {
|
||
enableDefaultPackages = true;
|
||
packages = with pkgs; [
|
||
noto-fonts
|
||
noto-fonts-cjk-sans
|
||
noto-fonts-cjk-serif
|
||
noto-fonts-color-emoji
|
||
roboto-mono
|
||
];
|
||
};
|
||
services.openssh.enable = true;
|
||
services.openssh.settings.PermitRootLogin = "prohibit-password";
|
||
services.openssh.settings.PasswordAuthentication = false;
|
||
services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ];
|
||
services.openssh.ports = [ 12022 ];
|
||
systemd.targets.sleep.enable = false;
|
||
systemd.targets.suspend.enable = false;
|
||
systemd.targets.hibernate.enable = false;
|
||
systemd.targets.hybrid-sleep.enable = false;
|
||
fileSystems."/gshare" = {
|
||
device = "192.168.16.1:/data1/share";
|
||
fsType = "nfs";
|
||
};
|
||
## users
|
||
users.mutableUsers = true;
|
||
users.users.root.openssh.authorizedKeys.keys = [
|
||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyZILj+GxTUhdCgz2w1TxQ+aTcggnOJIb84qA4u271S asz258-17ac-bm-v0"
|
||
];
|
||
users.users.root.shell = pkgs.zsh;
|
||
users.users.nasp = {
|
||
isNormalUser = true;
|
||
createHome = true;
|
||
group = "nasp";
|
||
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
|
||
packages = with pkgs; [
|
||
firefox
|
||
];
|
||
shell = pkgs.zsh;
|
||
hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
|
||
};
|
||
users.groups.nasp = {};
|
||
## desktop
|
||
services.xserver = {
|
||
enable = true;
|
||
displayManager.gdm.enable = true;
|
||
desktopManager.gnome.enable = true;
|
||
xkb.layout = "us";
|
||
};
|
||
}
|
||
(lib.mkIf (config.networking.hostId != null) {
|
||
boot.supportedFilesystems = [ "zfs" ];
|
||
boot.zfs.forceImportRoot = false;
|
||
services.zfs.autoScrub.enable = true;
|
||
services.zfs.autoSnapshot.enable = true;
|
||
})
|
||
# nvidia
|
||
(lib.mkIf (nvidiaCfg.enable) {
|
||
nixpkgs.config.nvidia.acceptLicense = true;
|
||
hardware.nvidia = {
|
||
modesetting.enable = false;
|
||
powerManagement.enable = false;
|
||
powerManagement.finegrained = false;
|
||
open = false;
|
||
nvidiaSettings = true;
|
||
};
|
||
hardware.graphics = {
|
||
enable = true;
|
||
enable32Bit = true;
|
||
};
|
||
hardware.nvidia-container-toolkit.enable = true;
|
||
services.xserver.videoDrivers = [ "nvidia" ];
|
||
})
|
||
# docker
|
||
(lib.mkIf (dockerCfg.enable) {
|
||
virtualisation.docker = {
|
||
enable = true;
|
||
package = pkgs.docker_25;
|
||
daemon.settings = {
|
||
ipv6 = true;
|
||
fixed-cidr-v6 = "fddd:d0c1:1::/64";
|
||
experimental = true;
|
||
ip6tables = true;
|
||
live-restore = false;
|
||
};
|
||
};
|
||
environment.systemPackages = with pkgs; [
|
||
dnew
|
||
];
|
||
})
|
||
# nginx
|
||
(lib.mkIf (nginxCfg.enable) {
|
||
services.nginx = {
|
||
enable = true;
|
||
virtualHosts."default" = {
|
||
serverName = "_";
|
||
default = true;
|
||
locations."/" = {
|
||
return = "404";
|
||
};
|
||
};
|
||
virtualHosts."code-server" = {
|
||
serverName = "proxy.nasp.fit";
|
||
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
|
||
proxyWebsockets = true;
|
||
proxyPass = "http://unix:/home2/run/$1.sock";
|
||
extraConfig = ''
|
||
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
|
||
proxy_set_header Accept-Encoding gzip;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
systemd.timers."home2-sockets-clean" = {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "0:0:0";
|
||
Unit = "home2-sockets-clean.service";
|
||
};
|
||
};
|
||
systemd.services."home2-sockets-clean" = {
|
||
serviceConfig = {
|
||
Type = "simple";
|
||
User = "root";
|
||
};
|
||
script = ''
|
||
cd /home2/run
|
||
for i in ./*.sock; do
|
||
if ! socat -u OPEN:/dev/null UNIX-CONNECT:$i 1>/dev/null 2>&1; then
|
||
rm $i
|
||
fi
|
||
done
|
||
'';
|
||
path = [ pkgs.socat ];
|
||
};
|
||
})
|
||
# registry
|
||
(lib.mkIf (registryCfg.enable) {
|
||
systemd.timers."registry" = {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "*:0/5:0";
|
||
Unit = "registry.service";
|
||
};
|
||
};
|
||
systemd.services."registry" = {
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
User = "root";
|
||
};
|
||
script = builtins.readFile ./scripts/registry.sh;
|
||
path = with pkgs; [ git bash su shadow getent ];
|
||
};
|
||
security.sudo.extraConfig = ''
|
||
%nasp ALL = (root) NOPASSWD: ALL
|
||
'';
|
||
})
|
||
# sops-nix
|
||
## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8"
|
||
## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
|
||
## mkdir -p ~/.config/sops/age
|
||
## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt"
|
||
# telegraf
|
||
(lib.mkIf (telegrafCfg.enable) {
|
||
sops.secrets.telegraf = assert sopsCfg.enable; {};
|
||
services.telegraf = {
|
||
enable = true;
|
||
environmentFiles = [ "/run/secrets/telegraf" ];
|
||
extraConfig = {
|
||
agent = {
|
||
interval = "3s";
|
||
round_interval = true;
|
||
metric_batch_size = 1000;
|
||
metric_buffer_limit = 10000;
|
||
collection_jitter = "0s";
|
||
flush_interval = "30s";
|
||
flush_jitter = "3s";
|
||
precision = "0s";
|
||
hostname = assert (config.networking.hostName != ""); config.networking.hostName;
|
||
omit_hostname = false;
|
||
};
|
||
outputs.influxdb_v2 = {
|
||
urls = [ "\${INFLUX_URL}" ];
|
||
token = "\${INFLUX_TOKEN}";
|
||
organization = "nasp.fit";
|
||
bucket = "trash";
|
||
bucket_tag = "bucket";
|
||
exclude_bucket_tag = true;
|
||
timeout = "5s";
|
||
};
|
||
inputs.system = {
|
||
name_override = "system";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.cpu = {
|
||
percpu = false;
|
||
name_override = "cpu";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.kernel = {
|
||
name_override = "kernel";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.processes = {
|
||
name_override = "processes";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.mem = {
|
||
name_override = "memory";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.disk = {
|
||
ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"];
|
||
name_override = "disk";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "30s";
|
||
};
|
||
inputs.diskio = {
|
||
name_override = "diskio";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.net = {
|
||
name_override = "net";
|
||
ignore_protocol_stats = true;
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.smart = {
|
||
path_smartctl = "/run/current-system/sw/bin/smartctl";
|
||
use_sudo = true;
|
||
name_override = "smartctl";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "30s";
|
||
};
|
||
inputs.temp = {
|
||
name_override = "temperture";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
};
|
||
};
|
||
security.sudo.extraConfig = ''
|
||
%telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl
|
||
'';
|
||
})
|
||
(lib.mkIf (telegrafCfg.enable && nvidiaCfg.enable) {
|
||
services.telegraf.extraConfig = {
|
||
inputs.nvidia_smi = {
|
||
bin_path = "/run/current-system/sw/bin/nvidia-smi";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
};
|
||
})
|
||
# g series
|
||
(lib.mkIf (gCfg.enable) {
|
||
warnings = [ "To my knowledge, the g-series server is deprecated (2025-10-28)." ];
|
||
## network
|
||
networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
|
||
networking.nameservers = [ "192.168.16.1" ];
|
||
networking.firewall.extraInputRules = ''
|
||
ip saddr 192.168.16.0/24 accept
|
||
ip saddr 12.12.12.0/24 accept
|
||
'';
|
||
## packages and services
|
||
nasp.docker.enable = lib.mkDefault true;
|
||
nasp.nvidia.enable = lib.mkDefault true;
|
||
nasp.registry.enable = lib.mkDefault true;
|
||
nasp.nginx.enable = lib.mkDefault true;
|
||
nasp.sops.enable = lib.mkDefault true;
|
||
nasp.telegraf = {
|
||
enable = lib.mkDefault true;
|
||
bucket = "g-series";
|
||
};
|
||
services.ntp = {
|
||
enable = true;
|
||
servers = [ "192.168.16.1" ];
|
||
};
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth0Name != "") {
|
||
systemd.network.networks."10-eth0" = {
|
||
matchConfig.Name = gCfg.eth0Name;
|
||
networkConfig = {
|
||
DHCP = "no";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
address = [ "192.168.16.${builtins.toString (gCfg.serial + 100)}/24" ];
|
||
routes = [
|
||
{
|
||
Gateway = "192.168.16.1";
|
||
GatewayOnLink = true;
|
||
Metric = 90;
|
||
}
|
||
];
|
||
};
|
||
networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth1Name != "") {
|
||
systemd.network.networks."10-eth1" = {
|
||
matchConfig.Name = gCfg.eth1Name;
|
||
networkConfig = {
|
||
DHCP = "yes";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth2Name != "") {
|
||
systemd.network.networks."10-eth2" = {
|
||
matchConfig.Name = gCfg.eth2Name;
|
||
address = [ "12.12.12.${builtins.toString (gCfg.serial + 100)}/24" ];
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
networking.rxe = {
|
||
enable = true;
|
||
interfaces = [ "${gCfg.eth2Name}" ];
|
||
};
|
||
})
|
||
(lib.mkIf (cCfg.enable) {
|
||
networking.hostName = "c" + (builtins.toString cCfg.serial);
|
||
networking.nameservers = [ "192.168.20.1" ];
|
||
zramSwap.enable = true;
|
||
nasp.docker.enable = lib.mkDefault true;
|
||
nasp.registry.enable = lib.mkDefault true;
|
||
nasp.nginx.enable = lib.mkDefault true;
|
||
systemd.network.networks."10-eth-lan" = {
|
||
matchConfig.Name = cCfg.ethLanName;
|
||
networkConfig = {
|
||
DHCP = "no";
|
||
IPv6AcceptRA = false;
|
||
};
|
||
address = [
|
||
"192.168.20.${builtins.toString (cCfg.serial + 100)}/24"
|
||
"fd01:da8:bf:14::${decToHex (cCfg.serial + 100)}/64"
|
||
];
|
||
routes = [
|
||
{ Gateway = "192.168.20.1"; }
|
||
{ Gateway = "fd01:da8:bf:14::1"; }
|
||
];
|
||
};
|
||
networking.interfaces.${cCfg.ethLanName}.wakeOnLan.enable = true;
|
||
})
|
||
(lib.mkIf (cCfg.enable && cCfg.ethRDMAName != "") {
|
||
systemd.network.networks."10-eth-rdma" = {
|
||
matchConfig.Name = cCfg.ethRDMAName;
|
||
address = [ "10.10.10.${builtins.toString (cCfg.serial + 100)}/24" ];
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
networking.rxe = {
|
||
enable = true;
|
||
interfaces = [ "${cCfg.ethRDMAName}" ];
|
||
};
|
||
environment.systemPackages = [ pkgs.rdma-core ];
|
||
networking.firewall.extraInputRules = ''
|
||
ip saddr 10.10.10.0/24 accept
|
||
'';
|
||
})
|
||
]);
|
||
}
|