feat: set up c-series (WIP)

This commit is contained in:
Dict Xiong 2025-10-28 01:51:00 +08:00
parent ed9f6d2165
commit b2b5deb81d
9 changed files with 162 additions and 201 deletions

View File

@ -25,6 +25,9 @@
"g18-next"
"web-server"
];
nasp = (import ./modules/nasp {
lib = nixpkgs.lib;
});
in
{
nixosConfigurations = {
@ -35,6 +38,7 @@
specialArgs = { inherit inputs; };
modules = [
./hosts/${host}/configuration.nix
nasp.nixosModules.main
sops-nix.nixosModules.sops
];
};

View File

@ -4,7 +4,6 @@
imports =
[
./hardware-configuration.nix
../modules/nasp.nix
];
nasp = {
enable = true;

View File

@ -1,81 +0,0 @@
{ inputs, config, lib, pkgs, ... }:
{
imports =
[
./hardware-configuration.nix
../modules/nasp.nix
];
nasp = {
enable = true;
};
## nix
nix.settings.experimental-features = [ "nix-command" "flakes" ];
## hardware and system
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
time.hardwareClockInLocalTime = true;
## network
services.resolved.enable = true;
networking.nameservers = [
"166.111.8.29"
"166.111.8.28"
];
networking.networkmanager.enable = false;
networking.useDHCP = false;
systemd.network.enable = true;
networking.firewall.allowedTCPPorts = [ 80 443 12022 ];
## packages and services
nixpkgs.config.allowUnfree = true;
## desktop
services.xserver = {
enable = true;
displayManager.gdm.enable = true;
desktopManager.gnome.enable = true;
xkb.layout = "us";
};
# networking
networking.hostName = "web-server";
systemd.network.networks."10-thunet" = {
matchConfig.Name = "enp3s0f0";
networkConfig = {
DHCP = "no";
IPv6AcceptRA = false;
};
linkConfig = {
MACAddress = "58:97:bd:68:22:c2";
};
address = [ "166.111.68.109/28" ];
routes = [
{
routeConfig = {
Gateway = "166.111.68.97";
GatewayOnLink = true;
Metric = 90;
};
}
];
};
# service: nginx
services.nginx = {
enable = true;
virtualHosts."nasp.cs.tsinghua.edu.cn" = {
http2 = true;
addSSL = true;
enableACME = true;
locations."/" = {
root = "/data0/var/www/www";
};
};
};
security.acme = {
acceptTerms = true;
defaults.email = "xd21@mails.tsinghua.edu.cn";
};
system.stateVersion = "24.05";
}

View File

@ -1,39 +0,0 @@
# Do not modify this file! It was generated by nixos-generate-config
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "ehci_pci" "ahci" "mpt3sas" "usb_storage" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-intel" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/4bc05f99-244b-4af9-a751-4eb80199857c";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/75A7-1EBE";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp3s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp3s0f1.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

7
modules/nasp/default.nix Normal file
View File

@ -0,0 +1,7 @@
{ lib, ... }:
{
nixosModules.main = {config, pkgs, lib, ... }: (import ./main.nix {
inherit config pkgs lib;
});
}

12
modules/nasp/flake.nix Normal file
View File

@ -0,0 +1,12 @@
{
description = "Config Module for NASP's NixOS System";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/master";
};
outputs = { self, nixpkgs, ... }@inputs: rec {
nixosModules.main = {config, pkgs, lib, ... } : (import ./main.nix {
inherit config pkgs lib;
});
};
}

View File

@ -9,7 +9,22 @@ let
sopsCfg = mainCfg.sops;
telegrafCfg = mainCfg.telegraf;
gCfg = mainCfg.gSeries;
cCfg = mainCfg.cSeries;
dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
decToHex =
let
intToHex = [
"0" "1" "2" "3" "4" "5" "6" "7" "8" "9"
"a" "b" "c" "d" "e" "f"
];
toHex' = q: a:
if q > 0
then (toHex'
(q / 16)
((lib.elemAt intToHex (lib.mod q 16)) + a))
else a;
in
v: toHex' v "";
in
{
options.nasp = {
@ -22,7 +37,6 @@ in
};
nginx = {
enable = lib.mkEnableOption "nginx web server";
enableCodeServer = lib.mkEnableOption "proxy code server in docker";
};
registry = {
enable = lib.mkEnableOption "the nasp registry";
@ -41,7 +55,7 @@ in
gSeries = {
enable = lib.mkEnableOption "the g-series server configurations";
serial = lib.mkOption {
type = lib.types.int;
type = lib.types.ints.u8;
description = "Serial of the machine (gX)";
};
eth0Name = lib.mkOption {
@ -60,6 +74,23 @@ in
description = "Name of the RoCE NIC";
};
};
cSeries = {
enable = lib.mkEnableOption "cpu server configurations";
serial = lib.mkOption {
type = lib.types.ints.u8;
description = "Serial of the machine (cX)";
};
ethLanName = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of the LAN NIC";
};
ethRDMAName = lib.mkOption {
type = lib.types.str;
default = "";
description = "Name of the RDMA NIC";
};
};
};
# inplementation
@ -67,12 +98,14 @@ in
# base
{
## nix
nix.settings.experimental-features = [ "nix-command" "flakes" ];
nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
nix = {
settings.experimental-features = [ "nix-command" "flakes" ];
settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
optimise.automatic = true;
};
## hardware and system
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
time.hardwareClockInLocalTime = true;
i18n.defaultLocale = "C.UTF-8";
i18n.extraLocaleSettings = lib.mkDefault {
LC_ADDRESS = "zh_CN.UTF-8";
@ -87,26 +120,56 @@ in
};
time.timeZone = lib.mkDefault "Asia/Shanghai";
## network
services.avahi.enable = true;
boot.kernel.sysctl = {
"net.core.default_qdisc" = "fq";
"net.ipv4.tcp_congestion_control" = "bbr";
};
services.resolved.enable = true;
networking.networkmanager.enable = false;
networking.useDHCP = false;
systemd.network.enable = true;
networking.firewall.allowedTCPPorts = [ 12022 ];
networking.nftables.enable = true;
networking.extraHosts = ''
192.168.16.1 ssh.nasp.fit git.nasp.fit jump.nasp.fit
192.168.16.115 g15.nasp g15 lm1
192.168.16.116 g16.nasp g16 lm2
192.168.20.101 c1 c1.nasp
192.168.20.102 c2 c2.nasp
'';
## packages and services
nixpkgs.config.allowUnfree = true;
environment.systemPackages = with pkgs; [
bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man
openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh
# extended
acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw
lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat
sysstat tcpdump unzip usbutils virt-what zip
# full
bash cmake curl dialog dig dmidecode e2fsprogs ethtool expect fd file fzf gcc git gnumake
gnupg htop iftop inetutils iotop iproute2 iputils jq less lrzsz lshw lsof man mtr
nettools nmap pciutils openssh openssl p7zip python3 ripgrep socat sops sudo tcpdump tmux
unzip usbutils util-linux vim wget zip zsh zssh zstd
acpi asciinema atop bat bridge-utils btop conntrack-tools dos2unix ffmpeg fish git-lfs
imagemagick iptstate killall libwebp ndisc6 netcat-gnu nethogs nix-diff nvme-cli pstree
pwgen smartmontools sysstat tldr virt-what
wireshark zmap
];
programs.zsh.enable = true;
programs.nix-ld.enable = true;
programs.git.lfs.enable = true;
services.cron.enable = true;
services.locate = {
enable = true;
package = pkgs.plocate;
};
fonts = {
enableDefaultPackages = true;
packages = with pkgs; [
noto-fonts
noto-fonts-cjk-sans
noto-fonts-cjk-serif
noto-fonts-color-emoji
vista-fonts
vista-fonts-chs
roboto-mono
];
};
services.openssh.enable = true;
services.openssh.settings.PermitRootLogin = "prohibit-password";
services.openssh.settings.PasswordAuthentication = false;
@ -116,6 +179,10 @@ in
systemd.targets.suspend.enable = false;
systemd.targets.hibernate.enable = false;
systemd.targets.hybrid-sleep.enable = false;
fileSystems."/gshare" = {
device = "192.168.16.1:/data1/share";
fsType = "nfs";
};
## users
users.mutableUsers = true;
users.users.root.openssh.authorizedKeys.keys = [
@ -124,13 +191,6 @@ in
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyZILj+GxTUhdCgz2w1TxQ+aTcggnOJIb84qA4u271S asz258-17ac-bm-v0"
];
users.users.root.shell = pkgs.zsh;
system.activationScripts.dotfilesSetup.text = ''
if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then
source ${config.system.build.setEnvironment}
rm -rf ~/dotfiles
bash <(curl -fsSL dotfiles.cn)
fi
'';
users.users.nasp = {
isNormalUser = true;
createHome = true;
@ -139,6 +199,7 @@ in
packages = with pkgs; [
firefox
];
shell = pkgs.zsh;
hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
};
users.groups.nasp = {};
@ -154,12 +215,12 @@ in
boot.supportedFilesystems = [ "zfs" ];
boot.zfs.forceImportRoot = false;
services.zfs.autoScrub.enable = true;
services.zfs.autoSnapshot.enable = true;
})
# nvidia
(lib.mkIf (nvidiaCfg.enable) {
nixpkgs.config.nvidia.acceptLicense = true;
hardware.nvidia = {
package = config.boot.kernelPackages.nvidiaPackages.legacy_470;
modesetting.enable = false;
powerManagement.enable = false;
powerManagement.finegrained = false;
@ -171,12 +232,6 @@ in
enable32Bit = true;
};
hardware.nvidia-container-toolkit.enable = true;
systemd.services.nvidia-container-toolkit-cdi-generator = {
path = [ pkgs.jq pkgs.moreutils ];
postStart = ''
jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json
'';
};
services.xserver.videoDrivers = [ "nvidia" ];
})
# docker
@ -207,20 +262,19 @@ in
return = "404";
};
};
};
})
(lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) {
services.nginx.virtualHosts."code-server" = {
serverName = "proxy.nasp.fit";
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
extraConfig = ''
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
proxy_pass "http://unix:/home2/run/$1.sock";
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection upgrade;
proxy_set_header Accept-Encoding gzip;
'';
virtualHosts."code-server" = {
serverName = "proxy.nasp.fit";
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
proxyWebsockets = true;
extraConfig = ''
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
proxy_pass "http://unix:/home2/run/$1.sock";
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection upgrade;
proxy_set_header Accept-Encoding gzip;
'';
};
};
};
})
@ -250,10 +304,6 @@ in
## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
## mkdir -p ~/.config/sops/age
## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt"
(lib.mkIf sopsCfg.enable {
sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml;
sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
})
# telegraf
(lib.mkIf (telegrafCfg.enable) {
sops.secrets.telegraf = assert sopsCfg.enable; {};
@ -349,46 +399,19 @@ in
})
# g series
(lib.mkIf (gCfg.enable) {
warnings = [ "To my knowledge, the g-series server is deprecated (2025-10-28)." ];
## network
networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
networking.search = [ "" ];
networking.nameservers = [ "192.168.16.1" ];
networking.extraHosts = ''
192.168.16.1 nasp.fit git.nasp.fit
192.168.16.101 g1.nasp g1
192.168.16.102 g2.nasp g2
192.168.16.103 g3.nasp g3
192.168.16.104 g4.nasp g4
192.168.16.105 g5.nasp g5
192.168.16.106 g6.nasp g6
192.168.16.107 g7.nasp g7
192.168.16.108 g8.nasp g8
192.168.16.109 g9.nasp g9
192.168.16.110 g10.nasp g10
192.168.16.111 g11.nasp g11
192.168.16.112 g12.nasp g12
192.168.16.113 g13.nasp g13
192.168.16.114 g14.nasp g14
192.168.16.115 g15.nasp g15
192.168.16.116 g16.nasp g16
192.168.16.117 g17.nasp g17
192.168.16.118 g18.nasp g18
192.168.16.119 g19.nasp g19
networking.firewall.extraInputRules = ''
ip saddr 192.168.16.0/24 accept
ip saddr 12.12.12.0/24 accept
'';
networking.firewall.extraCommands = ''
iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT
iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT
'';
fileSystems."/gshare" = {
device = "192.168.16.1:/data1/share";
fsType = "nfs";
};
## packages and services
nasp.docker.enable = lib.mkDefault true;
nasp.nvidia.enable = lib.mkDefault true;
nasp.registry.enable = lib.mkDefault true;
nasp.nginx.enable = lib.mkDefault true;
nasp.nginx.enableCodeServer = lib.mkDefault true;
nasp.sops.enable = lib.mkDefault true;
nasp.telegraf = {
enable = lib.mkDefault true;
@ -413,11 +436,6 @@ in
GatewayOnLink = true;
Metric = 90;
}
{
Gateway = "fd01:da8:bf:300::1";
GatewayOnLink = true;
Metric = 90;
}
];
};
networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
@ -443,5 +461,46 @@ in
interfaces = [ "${gCfg.eth2Name}" ];
};
})
(lib.mkIf (cCfg.enable) {
networking.hostName = "c" + (builtins.toString cCfg.serial);
networking.nameservers = [ "192.168.20.1" ];
networking.firewall.extraInputRules = ''
ip saddr 192.168.20.0/24 accept
ip6 saddr fd01:da8:bf:14::/64 accept
'';
nasp.docker.enable = lib.mkDefault true;
nasp.registry.enable = lib.mkDefault true;
nasp.nginx.enable = lib.mkDefault true;
systemd.network.networks."10-eth-lan" = {
matchConfig.Name = cCfg.ethLanName;
networkConfig = {
DHCP = "no";
IPv6AcceptRA = false;
};
address = [
"192.168.20.${builtins.toString (cCfg.serial + 100)}/24"
"fd01:da8:bf:14::${decToHex (cCfg.serial + 100)}/64"
];
routes = [
{ Gateway = "192.168.20.1"; }
{ Gateway = "fd01:da8:bf:14::1"; }
];
};
networking.interfaces.${cCfg.ethLanName}.wakeOnLan.enable = true;
})
(lib.mkIf (cCfg.enable && cCfg.ethRDMAName != "") {
systemd.network.networks."10-eth-rdma" = {
matchConfig.Name = cCfg.ethRDMAName;
address = [ "12.12.12.${builtins.toString (cCfg.serial + 100)}/24" ];
linkConfig.RequiredForOnline = "no";
};
networking.rxe = {
enable = true;
interfaces = [ "${cCfg.ethRDMAName}" ];
};
networking.firewall.extraInputRules = ''
ip saddr 12.12.12.0/24 accept
'';
})
]);
}