325 lines
11 KiB
Nix
325 lines
11 KiB
Nix
{ config, pkgs, lib, ... }:
|
||
|
||
let
|
||
mainCfg = config.nasp;
|
||
nvidiaCfg = mainCfg.nvidia;
|
||
dockerCfg = mainCfg.docker;
|
||
nginxCfg = mainCfg.nginx;
|
||
registryCfg = mainCfg.registry;
|
||
gCfg = mainCfg.gSeries;
|
||
dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
|
||
in
|
||
{
|
||
options.nasp = {
|
||
enable = lib.mkEnableOption "the nasp server configurations";
|
||
nvidia = {
|
||
enable = lib.mkEnableOption "nvidia gpu support";
|
||
};
|
||
docker = {
|
||
enable = lib.mkEnableOption "docker runtime";
|
||
};
|
||
nginx = {
|
||
enable = lib.mkEnableOption "nginx web server";
|
||
enableCodeServer = lib.mkEnableOption "proxy code server in docker";
|
||
};
|
||
registry = {
|
||
enable = lib.mkEnableOption "the nasp registry";
|
||
};
|
||
gSeries = {
|
||
enable = lib.mkEnableOption "the g-series server configurations";
|
||
serial = lib.mkOption {
|
||
type = lib.types.int;
|
||
description = "Serial of the machine (gX)";
|
||
};
|
||
eth0Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = "Name of eth0 (192.168.16.0/24)";
|
||
};
|
||
eth1Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = "Name of eth1 (thunet)";
|
||
};
|
||
eth2Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = "Name of the RoCE NIC";
|
||
};
|
||
};
|
||
};
|
||
|
||
# inplementation
|
||
config = lib.mkIf mainCfg.enable (lib.mkMerge [
|
||
# base
|
||
{
|
||
## nix
|
||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||
nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
|
||
## hardware and system
|
||
boot.loader.systemd-boot.enable = true;
|
||
boot.loader.efi.canTouchEfiVariables = true;
|
||
time.hardwareClockInLocalTime = true;
|
||
i18n.defaultLocale = "C.UTF-8";
|
||
i18n.extraLocaleSettings = lib.mkDefault {
|
||
LC_ADDRESS = "zh_CN.UTF-8";
|
||
LC_IDENTIFICATION = "zh_CN.UTF-8";
|
||
LC_MEASUREMENT = "zh_CN.UTF-8";
|
||
LC_MONETARY = "zh_CN.UTF-8";
|
||
LC_NAME = "zh_CN.UTF-8";
|
||
LC_NUMERIC = "zh_CN.UTF-8";
|
||
LC_PAPER = "zh_CN.UTF-8";
|
||
LC_TELEPHONE = "zh_CN.UTF-8";
|
||
LC_TIME = "zh_CN.UTF-8";
|
||
};
|
||
time.timeZone = lib.mkDefault "Asia/Shanghai";
|
||
## network
|
||
services.resolved.enable = true;
|
||
networking.networkmanager.enable = false;
|
||
networking.useDHCP = false;
|
||
systemd.network.enable = true;
|
||
networking.firewall.allowedTCPPorts = [ 12022 ];
|
||
## packages and services
|
||
nixpkgs.config.allowUnfree = true;
|
||
environment.systemPackages = with pkgs; [
|
||
bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man
|
||
openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh
|
||
# extended
|
||
acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw
|
||
lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat
|
||
sysstat tcpdump unzip usbutils virt-what zip
|
||
# full
|
||
wireshark zmap
|
||
];
|
||
programs.zsh.enable = true;
|
||
services.cron.enable = true;
|
||
services.openssh.enable = true;
|
||
services.openssh.settings.PermitRootLogin = "prohibit-password";
|
||
services.openssh.settings.PasswordAuthentication = false;
|
||
services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ];
|
||
services.openssh.ports = [ 12022 ];
|
||
systemd.targets.sleep.enable = false;
|
||
systemd.targets.suspend.enable = false;
|
||
systemd.targets.hibernate.enable = false;
|
||
systemd.targets.hybrid-sleep.enable = false;
|
||
## users
|
||
users.mutableUsers = true;
|
||
users.users.root.openssh.authorizedKeys.keys = [
|
||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhS4voo3K/Dvzqckr0bouO1WkCI5XxswstHWnuuyKBz ltp1-bd"
|
||
];
|
||
users.users.root.shell = pkgs.zsh;
|
||
system.activationScripts.dotfilesSetup.text = ''
|
||
if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then
|
||
source ${config.system.build.setEnvironment}
|
||
rm -rf ~/dotfiles
|
||
bash <(curl -fsSL dotfiles.cn)
|
||
fi
|
||
'';
|
||
users.users.nasp = {
|
||
isNormalUser = true;
|
||
createHome = true;
|
||
group = "nasp";
|
||
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
|
||
packages = with pkgs; [
|
||
firefox
|
||
];
|
||
hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
|
||
};
|
||
users.groups.nasp = {};
|
||
## desktop
|
||
services.xserver = {
|
||
enable = true;
|
||
displayManager.gdm.enable = true;
|
||
desktopManager.gnome.enable = true;
|
||
xkb.layout = "us";
|
||
};
|
||
}
|
||
(lib.mkIf (config.networking.hostId != null) {
|
||
boot.supportedFilesystems = [ "zfs" ];
|
||
boot.zfs.forceImportRoot = false;
|
||
services.zfs.autoScrub.enable = true;
|
||
})
|
||
# nvidia
|
||
(lib.mkIf (nvidiaCfg.enable) {
|
||
nixpkgs.config.nvidia.acceptLicense = true;
|
||
hardware.nvidia = {
|
||
package = config.boot.kernelPackages.nvidiaPackages.legacy_470;
|
||
modesetting.enable = false;
|
||
powerManagement.enable = false;
|
||
powerManagement.finegrained = false;
|
||
open = false;
|
||
nvidiaSettings = true;
|
||
};
|
||
hardware.opengl = {
|
||
enable = true;
|
||
driSupport = true;
|
||
driSupport32Bit = true;
|
||
};
|
||
hardware.nvidia-container-toolkit.enable = true;
|
||
systemd.services.nvidia-container-toolkit-cdi-generator = {
|
||
path = [ pkgs.jq pkgs.moreutils ];
|
||
postStart = ''
|
||
jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json
|
||
'';
|
||
};
|
||
services.xserver.videoDrivers = [ "nvidia" ];
|
||
})
|
||
# docker
|
||
(lib.mkIf (dockerCfg.enable) {
|
||
virtualisation.docker = {
|
||
enable = true;
|
||
package = pkgs.docker_25;
|
||
daemon.settings = {
|
||
ipv6 = true;
|
||
fixed-cidr-v6 = "fddd:d0c1:1::/64";
|
||
experimental = true;
|
||
ip6tables = true;
|
||
live-restore = false;
|
||
};
|
||
};
|
||
environment.systemPackages = with pkgs; [
|
||
dnew
|
||
];
|
||
})
|
||
# nginx
|
||
(lib.mkIf (nginxCfg.enable) {
|
||
services.nginx = {
|
||
enable = true;
|
||
virtualHosts."default" = {
|
||
serverName = "_";
|
||
default = true;
|
||
locations."/" = {
|
||
return = "404";
|
||
};
|
||
};
|
||
};
|
||
})
|
||
(lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) {
|
||
services.nginx.virtualHosts."code-server" = {
|
||
serverName = "proxy.nasp.fit";
|
||
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
|
||
extraConfig = ''
|
||
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
|
||
proxy_pass "http://unix:/home2/run/$1.sock";
|
||
proxy_set_header Host $host;
|
||
proxy_set_header Upgrade $http_upgrade;
|
||
proxy_set_header Connection upgrade;
|
||
proxy_set_header Accept-Encoding gzip;
|
||
'';
|
||
};
|
||
};
|
||
})
|
||
# registry
|
||
(lib.mkIf (registryCfg.enable) {
|
||
systemd.timers."registry" = {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "*:0/5:0";
|
||
Unit = "registry.service";
|
||
};
|
||
};
|
||
systemd.services."registry" = {
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
User = "root";
|
||
};
|
||
script = builtins.readFile ./scripts/registry.sh;
|
||
path = with pkgs; [ git bash su shadow getent ];
|
||
};
|
||
security.sudo.extraConfig = ''
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/docker
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/reboot
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/whoami
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/nvidia-smi
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/shutdown
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/ip
|
||
'';
|
||
})
|
||
# g series
|
||
(lib.mkIf (gCfg.enable) (let
|
||
ipSuffix = (builtins.toString (gCfg.serial + 100)); in {
|
||
## network
|
||
networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
|
||
networking.search = [ "nasp" ];
|
||
networking.nameservers = [ "192.168.16.118" ];
|
||
systemd.network.networks."10-eth0" = {
|
||
matchConfig.Name = gCfg.eth0Name;
|
||
networkConfig = {
|
||
DHCP = "no";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
address = [ "192.168.16.${ipSuffix}/24" ];
|
||
routes = [
|
||
{
|
||
routeConfig = {
|
||
Gateway = "192.168.16.118";
|
||
GatewayOnLink = true;
|
||
Metric = 90;
|
||
};
|
||
}
|
||
{
|
||
routeConfig = {
|
||
Gateway = "2001:da8:bf:300::1";
|
||
GatewayOnLink = true;
|
||
Metric = 90;
|
||
};
|
||
}
|
||
];
|
||
};
|
||
systemd.network.networks."10-eth1" = {
|
||
matchConfig.Name = gCfg.eth1Name;
|
||
networkConfig = {
|
||
DHCP = "yes";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
systemd.network.networks."10-eth2" = {
|
||
matchConfig.Name = gCfg.eth2Name;
|
||
address = [ "12.12.12.${ipSuffix}/24" ];
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
|
||
networking.extraHosts = ''
|
||
192.168.16.101 g1-nasp g1
|
||
192.168.16.102 g2-nasp g2
|
||
192.168.16.103 g3-nasp g3
|
||
192.168.16.104 g4-nasp g4
|
||
192.168.16.105 g5-nasp g5
|
||
192.168.16.106 g6-nasp g6
|
||
192.168.16.107 g7-nasp g7
|
||
192.168.16.108 g8-nasp g8
|
||
192.168.16.109 g9-nasp g9
|
||
192.168.16.110 g10-nasp g10
|
||
192.168.16.111 g11-nasp g11
|
||
192.168.16.112 g12-nasp g12
|
||
192.168.16.113 g13-nasp g13
|
||
192.168.16.114 g14-nasp g14
|
||
192.168.16.115 g15-nasp g15
|
||
192.168.16.116 g16-nasp g16
|
||
192.168.16.117 g17-nasp g17
|
||
192.168.16.118 g18-nasp g18 nasp.fit git.nasp.fit
|
||
192.168.16.119 g19-nasp g19
|
||
192.168.16.120 g20-nasp g20
|
||
'';
|
||
networking.firewall.extraCommands = ''
|
||
iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT
|
||
iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT
|
||
'';
|
||
networking.rxe = {
|
||
enable = true;
|
||
interfaces = [ "${gCfg.eth2Name}" ];
|
||
};
|
||
fileSystems."/share" = {
|
||
device = "192.168.16.118:/data0/share";
|
||
fsType = "nfs";
|
||
};
|
||
## packages and services
|
||
nasp.docker.enable = true;
|
||
nasp.nvidia.enable = true;
|
||
nasp.registry.enable = true;
|
||
nasp.nginx.enable = true;
|
||
nasp.nginx.enableCodeServer = true;
|
||
}))
|
||
]);
|
||
}
|