NASP/NixOS-Config
84
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: set up c-series (WIP)

Browse Source
This commit is contained in:
Dict Xiong
2025-10-28 01:51:00 +08:00
parent ed9f6d2165
commit b2b5deb81d
9 changed files with 162 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modules/nasp/default.nix Normal file
View File

@@ -0,0 +1,7 @@
{ lib, ... }:
{
nixosModules.main = {config, pkgs, lib, ... }: (import ./main.nix {
inherit config pkgs lib;
});
}

12
modules/nasp/flake.nix Normal file
View File

@@ -0,0 +1,12 @@
{
description = "Config Module for NASP's NixOS System";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/master";
};
outputs = { self, nixpkgs, ... }@inputs: rec {
nixosModules.main = {config, pkgs, lib, ... } : (import ./main.nix {
inherit config pkgs lib;
});
};
}

506
modules/nasp/main.nix Normal file
View File

@@ -0,0 +1,506 @@
{ config, pkgs, lib, ... }:
let
mainCfg = config.nasp;
nvidiaCfg = mainCfg.nvidia;
dockerCfg = mainCfg.docker;
nginxCfg = mainCfg.nginx;
registryCfg = mainCfg.registry;
sopsCfg = mainCfg.sops;
telegrafCfg = mainCfg.telegraf;
gCfg = mainCfg.gSeries;
cCfg = mainCfg.cSeries;
dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
decToHex =
let
intToHex = [
"0" "1" "2" "3" "4" "5" "6" "7" "8" "9"
"a" "b" "c" "d" "e" "f"
];
toHex' = q: a:
if q > 0
then (toHex'
(q / 16)
((lib.elemAt intToHex (lib.mod q 16)) + a))
else a;
in
v: toHex' v "";
in
{
options.nasp = {
enable = lib.mkEnableOption "the nasp server configurations";
nvidia = {
enable = lib.mkEnableOption "nvidia gpu support";
};
docker = {
enable = lib.mkEnableOption "docker runtime";
};
nginx = {
enable = lib.mkEnableOption "nginx web server";
};
registry = {
enable = lib.mkEnableOption "the nasp registry";
};
sops = {
enable = lib.mkEnableOption "sops";
};
telegraf = {
enable = lib.mkEnableOption "telegraf";
bucket = lib.mkOption {
type = lib.types.str;
description = "target influxdb bucket";
default = "trash";
};
};
gSeries = {
enable = lib.mkEnableOption "the g-series server configurations";
serial = lib.mkOption {
type = lib.types.ints.u8;
description = "Serial of the machine (gX)";
};
eth0Name = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of eth0 (192.168.16.0/24)";
};
eth1Name = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of eth1 (thunet)";
};
eth2Name = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of the RoCE NIC";
};
};
cSeries = {
enable = lib.mkEnableOption "cpu server configurations";
serial = lib.mkOption {
type = lib.types.ints.u8;
description = "Serial of the machine (cX)";
};
ethLanName = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of the LAN NIC";
};
ethRDMAName = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of the RDMA NIC";
};
};
};
# inplementation
config = lib.mkIf mainCfg.enable (lib.mkMerge [
# base
{
## nix
nix = {
settings.experimental-features = [ "nix-command" "flakes" ];
settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
optimise.automatic = true;
};
## hardware and system
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
i18n.defaultLocale = "C.UTF-8";
i18n.extraLocaleSettings = lib.mkDefault {
LC_ADDRESS = "zh_CN.UTF-8";
LC_IDENTIFICATION = "zh_CN.UTF-8";
LC_MEASUREMENT = "zh_CN.UTF-8";
LC_MONETARY = "zh_CN.UTF-8";
LC_NAME = "zh_CN.UTF-8";
LC_NUMERIC = "zh_CN.UTF-8";
LC_PAPER = "zh_CN.UTF-8";
LC_TELEPHONE = "zh_CN.UTF-8";
LC_TIME = "zh_CN.UTF-8";
};
time.timeZone = lib.mkDefault "Asia/Shanghai";
## network
services.avahi.enable = true;
boot.kernel.sysctl = {
"net.core.default_qdisc" = "fq";
"net.ipv4.tcp_congestion_control" = "bbr";
};
services.resolved.enable = true;
networking.networkmanager.enable = false;
networking.useDHCP = false;
systemd.network.enable = true;
networking.firewall.allowedTCPPorts = [ 12022 ];
networking.nftables.enable = true;
networking.extraHosts = ''
192.168.16.1 ssh.nasp.fit git.nasp.fit jump.nasp.fit
192.168.16.115 g15.nasp g15 lm1
192.168.16.116 g16.nasp g16 lm2
192.168.20.101 c1 c1.nasp
192.168.20.102 c2 c2.nasp
'';
## packages and services
nixpkgs.config.allowUnfree = true;
environment.systemPackages = with pkgs; [
bash cmake curl dialog dig dmidecode e2fsprogs ethtool expect fd file fzf gcc git gnumake
gnupg htop iftop inetutils iotop iproute2 iputils jq less lrzsz lshw lsof man mtr
nettools nmap pciutils openssh openssl p7zip python3 ripgrep socat sops sudo tcpdump tmux
unzip usbutils util-linux vim wget zip zsh zssh zstd
acpi asciinema atop bat bridge-utils btop conntrack-tools dos2unix ffmpeg fish git-lfs
imagemagick iptstate killall libwebp ndisc6 netcat-gnu nethogs nix-diff nvme-cli pstree
pwgen smartmontools sysstat tldr virt-what
wireshark zmap
];
programs.zsh.enable = true;
programs.nix-ld.enable = true;
programs.git.lfs.enable = true;
services.cron.enable = true;
services.locate = {
enable = true;
package = pkgs.plocate;
};
fonts = {
enableDefaultPackages = true;
packages = with pkgs; [
noto-fonts
noto-fonts-cjk-sans
noto-fonts-cjk-serif
noto-fonts-color-emoji
vista-fonts
vista-fonts-chs
roboto-mono
];
};
services.openssh.enable = true;
services.openssh.settings.PermitRootLogin = "prohibit-password";
services.openssh.settings.PasswordAuthentication = false;
services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ];
services.openssh.ports = [ 12022 ];
systemd.targets.sleep.enable = false;
systemd.targets.suspend.enable = false;
systemd.targets.hibernate.enable = false;
systemd.targets.hybrid-sleep.enable = false;
fileSystems."/gshare" = {
device = "192.168.16.1:/data1/share";
fsType = "nfs";
};
## users
users.mutableUsers = true;
users.users.root.openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyZILj+GxTUhdCgz2w1TxQ+aTcggnOJIb84qA4u271S asz258-17ac-bm-v0"
];
users.users.root.shell = pkgs.zsh;
users.users.nasp = {
isNormalUser = true;
createHome = true;
group = "nasp";
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
packages = with pkgs; [
firefox
];
shell = pkgs.zsh;
hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
};
users.groups.nasp = {};
## desktop
services.xserver = {
enable = true;
displayManager.gdm.enable = true;
desktopManager.gnome.enable = true;
xkb.layout = "us";
};
}
(lib.mkIf (config.networking.hostId != null) {
boot.supportedFilesystems = [ "zfs" ];
boot.zfs.forceImportRoot = false;
services.zfs.autoScrub.enable = true;
services.zfs.autoSnapshot.enable = true;
})
# nvidia
(lib.mkIf (nvidiaCfg.enable) {
nixpkgs.config.nvidia.acceptLicense = true;
hardware.nvidia = {
modesetting.enable = false;
powerManagement.enable = false;
powerManagement.finegrained = false;
open = false;
nvidiaSettings = true;
};
hardware.graphics = {
enable = true;
enable32Bit = true;
};
hardware.nvidia-container-toolkit.enable = true;
services.xserver.videoDrivers = [ "nvidia" ];
})
# docker
(lib.mkIf (dockerCfg.enable) {
virtualisation.docker = {
enable = true;
package = pkgs.docker_25;
daemon.settings = {
ipv6 = true;
fixed-cidr-v6 = "fddd:d0c1:1::/64";
experimental = true;
ip6tables = true;
live-restore = false;
};
};
environment.systemPackages = with pkgs; [
dnew
];
})
# nginx
(lib.mkIf (nginxCfg.enable) {
services.nginx = {
enable = true;
virtualHosts."default" = {
serverName = "_";
default = true;
locations."/" = {
return = "404";
};
};
virtualHosts."code-server" = {
serverName = "proxy.nasp.fit";
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
proxyWebsockets = true;
extraConfig = ''
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
proxy_pass "http://unix:/home2/run/$1.sock";
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection upgrade;
proxy_set_header Accept-Encoding gzip;
'';
};
};
};
})
# registry
(lib.mkIf (registryCfg.enable) {
systemd.timers."registry" = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "*:0/5:0";
Unit = "registry.service";
};
};
systemd.services."registry" = {
serviceConfig = {
Type = "oneshot";
User = "root";
};
script = builtins.readFile ./scripts/registry.sh;
path = with pkgs; [ git bash su shadow getent ];
};
security.sudo.extraConfig = ''
%nasp ALL = (root) NOPASSWD: ALL
'';
})
# sops-nix
## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8"
## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
## mkdir -p ~/.config/sops/age
## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt"
# telegraf
(lib.mkIf (telegrafCfg.enable) {
sops.secrets.telegraf = assert sopsCfg.enable; {};
services.telegraf = {
enable = true;
environmentFiles = [ "/run/secrets/telegraf" ];
extraConfig = {
agent = {
interval = "3s";
round_interval = true;
metric_batch_size = 1000;
metric_buffer_limit = 10000;
collection_jitter = "0s";
flush_interval = "30s";
flush_jitter = "3s";
precision = "0s";
hostname = assert (config.networking.hostName != ""); config.networking.hostName;
omit_hostname = false;
};
outputs.influxdb_v2 = {
urls = [ "\${INFLUX_URL}" ];
token = "\${INFLUX_TOKEN}";
organization = "nasp.fit";
bucket = "trash";
bucket_tag = "bucket";
exclude_bucket_tag = true;
timeout = "5s";
};
inputs.system = {
name_override = "system";
tags = { bucket = telegrafCfg.bucket; };
interval = "10s";
};
inputs.cpu = {
percpu = false;
name_override = "cpu";
tags = { bucket = telegrafCfg.bucket; };
};
inputs.kernel = {
name_override = "kernel";
tags = { bucket = telegrafCfg.bucket; };
interval = "10s";
};
inputs.processes = {
name_override = "processes";
tags = { bucket = telegrafCfg.bucket; };
interval = "10s";
};
inputs.mem = {
name_override = "memory";
tags = { bucket = telegrafCfg.bucket; };
interval = "10s";
};
inputs.disk = {
ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"];
name_override = "disk";
tags = { bucket = telegrafCfg.bucket; };
interval = "30s";
};
inputs.diskio = {
name_override = "diskio";
tags = { bucket = telegrafCfg.bucket; };
};
inputs.net = {
name_override = "net";
ignore_protocol_stats = true;
tags = { bucket = telegrafCfg.bucket; };
};
inputs.smart = {
path_smartctl = "/run/current-system/sw/bin/smartctl";
use_sudo = true;
name_override = "smartctl";
tags = { bucket = telegrafCfg.bucket; };
interval = "30s";
};
inputs.temp = {
name_override = "temperture";
tags = { bucket = telegrafCfg.bucket; };
};
};
};
security.sudo.extraConfig = ''
%telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl
'';
})
(lib.mkIf (telegrafCfg.enable && nvidiaCfg.enable) {
services.telegraf.extraConfig = {
inputs.nvidia_smi = {
bin_path = "/run/current-system/sw/bin/nvidia-smi";
tags = { bucket = telegrafCfg.bucket; };
};
};
})
# g series
(lib.mkIf (gCfg.enable) {
warnings = [ "To my knowledge, the g-series server is deprecated (2025-10-28)." ];
## network
networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
networking.nameservers = [ "192.168.16.1" ];
networking.firewall.extraInputRules = ''
ip saddr 192.168.16.0/24 accept
ip saddr 12.12.12.0/24 accept
'';
## packages and services
nasp.docker.enable = lib.mkDefault true;
nasp.nvidia.enable = lib.mkDefault true;
nasp.registry.enable = lib.mkDefault true;
nasp.nginx.enable = lib.mkDefault true;
nasp.sops.enable = lib.mkDefault true;
nasp.telegraf = {
enable = lib.mkDefault true;
bucket = "g-series";
};
services.ntp = {
enable = true;
servers = [ "192.168.16.1" ];
};
})
(lib.mkIf (gCfg.enable && gCfg.eth0Name != "") {
systemd.network.networks."10-eth0" = {
matchConfig.Name = gCfg.eth0Name;
networkConfig = {
DHCP = "no";
IPv6AcceptRA = true;
};
address = [ "192.168.16.${builtins.toString (gCfg.serial + 100)}/24" ];
routes = [
{
Gateway = "192.168.16.1";
GatewayOnLink = true;
Metric = 90;
}
];
};
networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
})
(lib.mkIf (gCfg.enable && gCfg.eth1Name != "") {
systemd.network.networks."10-eth1" = {
matchConfig.Name = gCfg.eth1Name;
networkConfig = {
DHCP = "yes";
IPv6AcceptRA = true;
};
linkConfig.RequiredForOnline = "no";
};
})
(lib.mkIf (gCfg.enable && gCfg.eth2Name != "") {
systemd.network.networks."10-eth2" = {
matchConfig.Name = gCfg.eth2Name;
address = [ "12.12.12.${builtins.toString (gCfg.serial + 100)}/24" ];
linkConfig.RequiredForOnline = "no";
};
networking.rxe = {
enable = true;
interfaces = [ "${gCfg.eth2Name}" ];
};
})
(lib.mkIf (cCfg.enable) {
networking.hostName = "c" + (builtins.toString cCfg.serial);
networking.nameservers = [ "192.168.20.1" ];
networking.firewall.extraInputRules = ''
ip saddr 192.168.20.0/24 accept
ip6 saddr fd01:da8:bf:14::/64 accept
'';
nasp.docker.enable = lib.mkDefault true;
nasp.registry.enable = lib.mkDefault true;
nasp.nginx.enable = lib.mkDefault true;
systemd.network.networks."10-eth-lan" = {
matchConfig.Name = cCfg.ethLanName;
networkConfig = {
DHCP = "no";
IPv6AcceptRA = false;
};
address = [
"192.168.20.${builtins.toString (cCfg.serial + 100)}/24"
"fd01:da8:bf:14::${decToHex (cCfg.serial + 100)}/64"
];
routes = [
{ Gateway = "192.168.20.1"; }
{ Gateway = "fd01:da8:bf:14::1"; }
];
};
networking.interfaces.${cCfg.ethLanName}.wakeOnLan.enable = true;
})
(lib.mkIf (cCfg.enable && cCfg.ethRDMAName != "") {
systemd.network.networks."10-eth-rdma" = {
matchConfig.Name = cCfg.ethRDMAName;
address = [ "12.12.12.${builtins.toString (cCfg.serial + 100)}/24" ];
linkConfig.RequiredForOnline = "no";
};
networking.rxe = {
enable = true;
interfaces = [ "${cCfg.ethRDMAName}" ];
};
networking.firewall.extraInputRules = ''
ip saddr 12.12.12.0/24 accept
'';
})
]);
}

79
modules/nasp/scripts/dnew Executable file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -e
trap 'echo "An error occurred. Please contact the manager." >&2' ERR
echo "================== dnew ==================="
echo "=== contact: xd21@mails.tsinghua.edu.cn ==="
test_not_empty()
{
if [[ -z "$1" ]]; then
echo $2 >&2
exit
fi
}
ask_for_yN()
{
if [[ "$DFS_QUIET" == "1" ]]; then
echo 0
else
read -p "$1 [yN]: " yn
case $yn in
[Yy]* ) echo 1;;
* ) echo 0;;
esac
fi
}
container_name=$(whoami)_$(date +%y%m%d-%H%M%S)
base_command="sudo docker run -d --net=host \\
-v /home2:/home2 -v /gshare:/gshare \\
--privileged \\
--security-opt seccomp=unconfined --ulimit memlock=-1:-1 \\
--restart=unless-stopped \\
--name $container_name"
if [[ "$EUID" == "0" ]]; then
echo "Please run as your own user (i.e., DO NOT sudo)."
exit 1
fi
ret=$(ask_for_yN "Use GPU?")
if [[ "$ret" == "1" ]]; then
base_command="$base_command \\
--device nvidia.com/gpu=all"
fi
ret=$(ask_for_yN "Use RDMA?")
if [[ "$ret" == "1" ]]; then
base_command="$base_command \\
--device=/dev/infiniband/uverbs0"
fi
image_name="git.nasp.fit/nasp/nasp-ubuntu"
base_command="$base_command \\
-it $image_name"
cmd="/etc/startup.sh"
base_command="$base_command $cmd"
echo "Will run:"
echo "====="
echo "$base_command"
echo "====="
ret=$(ask_for_yN "Start the container?")
if [[ "$ret" == "1" ]]; then
eval "$base_command"
echo "Container $container_name started. You can use the following command to get in:"
echo "sudo docker exec -it $container_name bash"
if [[ "$image_name" == "git.nasp.fit/nasp/nasp-ubuntu" ]]; then
sleep 1s
sudo docker exec -it $container_name code.sh
fi
else
echo "Aborted."
fi

12
modules/nasp/scripts/registry.sh Normal file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -ex
if [[ ! -e /opt/registry/scripts/testbed_cron.sh ]]; then
mkdir -p /opt
cd /opt
git clone https://git.nasp.fit/NASP/registry.git
fi
cd /opt/registry
git pull
bash /opt/registry/scripts/testbed_cron.sh