457 lines
16 KiB
Nix
457 lines
16 KiB
Nix
{ config, pkgs, lib, ... }:
|
||
|
||
let
|
||
mainCfg = config.nasp;
|
||
nvidiaCfg = mainCfg.nvidia;
|
||
dockerCfg = mainCfg.docker;
|
||
nginxCfg = mainCfg.nginx;
|
||
registryCfg = mainCfg.registry;
|
||
sopsCfg = mainCfg.sops;
|
||
telegrafCfg = mainCfg.telegraf;
|
||
gCfg = mainCfg.gSeries;
|
||
dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew));
|
||
in
|
||
{
|
||
options.nasp = {
|
||
enable = lib.mkEnableOption "the nasp server configurations";
|
||
nvidia = {
|
||
enable = lib.mkEnableOption "nvidia gpu support";
|
||
};
|
||
docker = {
|
||
enable = lib.mkEnableOption "docker runtime";
|
||
};
|
||
nginx = {
|
||
enable = lib.mkEnableOption "nginx web server";
|
||
enableCodeServer = lib.mkEnableOption "proxy code server in docker";
|
||
};
|
||
registry = {
|
||
enable = lib.mkEnableOption "the nasp registry";
|
||
};
|
||
sops = {
|
||
enable = lib.mkEnableOption "sops";
|
||
};
|
||
telegraf = {
|
||
enable = lib.mkEnableOption "telegraf";
|
||
bucket = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = "target influxdb bucket";
|
||
default = "trash";
|
||
};
|
||
};
|
||
gSeries = {
|
||
enable = lib.mkEnableOption "the g-series server configurations";
|
||
serial = lib.mkOption {
|
||
type = lib.types.int;
|
||
description = "Serial of the machine (gX)";
|
||
};
|
||
eth0Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of eth0 (192.168.16.0/24)";
|
||
};
|
||
eth1Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of eth1 (thunet)";
|
||
};
|
||
eth2Name = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
description = "Name of the RoCE NIC";
|
||
};
|
||
};
|
||
};
|
||
|
||
# inplementation
|
||
config = lib.mkIf mainCfg.enable (lib.mkMerge [
|
||
# base
|
||
{
|
||
## nix
|
||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||
nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ];
|
||
## hardware and system
|
||
boot.loader.systemd-boot.enable = true;
|
||
boot.loader.efi.canTouchEfiVariables = true;
|
||
time.hardwareClockInLocalTime = true;
|
||
i18n.defaultLocale = "C.UTF-8";
|
||
i18n.extraLocaleSettings = lib.mkDefault {
|
||
LC_ADDRESS = "zh_CN.UTF-8";
|
||
LC_IDENTIFICATION = "zh_CN.UTF-8";
|
||
LC_MEASUREMENT = "zh_CN.UTF-8";
|
||
LC_MONETARY = "zh_CN.UTF-8";
|
||
LC_NAME = "zh_CN.UTF-8";
|
||
LC_NUMERIC = "zh_CN.UTF-8";
|
||
LC_PAPER = "zh_CN.UTF-8";
|
||
LC_TELEPHONE = "zh_CN.UTF-8";
|
||
LC_TIME = "zh_CN.UTF-8";
|
||
};
|
||
time.timeZone = lib.mkDefault "Asia/Shanghai";
|
||
## network
|
||
services.resolved.enable = true;
|
||
networking.networkmanager.enable = false;
|
||
networking.useDHCP = false;
|
||
systemd.network.enable = true;
|
||
networking.firewall.allowedTCPPorts = [ 12022 ];
|
||
## packages and services
|
||
nixpkgs.config.allowUnfree = true;
|
||
environment.systemPackages = with pkgs; [
|
||
bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man
|
||
openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh
|
||
# extended
|
||
acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw
|
||
lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat
|
||
sysstat tcpdump unzip usbutils virt-what zip
|
||
# full
|
||
wireshark zmap
|
||
];
|
||
programs.zsh.enable = true;
|
||
services.cron.enable = true;
|
||
services.openssh.enable = true;
|
||
services.openssh.settings.PermitRootLogin = "prohibit-password";
|
||
services.openssh.settings.PasswordAuthentication = false;
|
||
services.openssh.authorizedKeysFiles = [ ".ssh/authorized_keys2" ];
|
||
services.openssh.ports = [ 12022 ];
|
||
systemd.targets.sleep.enable = false;
|
||
systemd.targets.suspend.enable = false;
|
||
systemd.targets.hibernate.enable = false;
|
||
systemd.targets.hybrid-sleep.enable = false;
|
||
## users
|
||
users.mutableUsers = true;
|
||
users.users.root.openssh.authorizedKeys.keys = [
|
||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUN7IXF4nlFcVfgHesgik3LIAiXlVMYJPm3yD13EVarQx5jqdBgk8Dwgkgf4rPO6MFpvIpinOyEO8zOS6HHQrCLZUv5yTFaDkUuB7eQ0EmpicGbmk9bHqj1HkOZxaobkpEfQUmFKYvkp4EexVw66sO0qfXvjHZ4H6yCAJLK5aUnKfgrE8tODzP82sU/mpJjW+Pq3uanNq754gaHwhxCIXG143/zp8qzBAeKe38xVqqDq9fTkG4hvzFvkRdS88i6l1z++0P3n0HGdOjtSg7P7fO7+7ZyPYr0gO5vB720Om/zxqPrGd9cicWi4P+aVKa+0ujWH/pqufWG6uCjKWHnBs7 sk0/piv/9a"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHLYgVj+NPino6sOmahULN7SbAMaVAgzqPfDjz2S8zDv pc1/windows"
|
||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhS4voo3K/Dvzqckr0bouO1WkCI5XxswstHWnuuyKBz ltp1-bd"
|
||
];
|
||
users.users.root.shell = pkgs.zsh;
|
||
system.activationScripts.dotfilesSetup.text = ''
|
||
if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then
|
||
source ${config.system.build.setEnvironment}
|
||
rm -rf ~/dotfiles
|
||
bash <(curl -fsSL dotfiles.cn)
|
||
fi
|
||
'';
|
||
users.users.nasp = {
|
||
isNormalUser = true;
|
||
createHome = true;
|
||
group = "nasp";
|
||
extraGroups = [ "wheel" ]; # Enable ‘sudo’ for the user.
|
||
packages = with pkgs; [
|
||
firefox
|
||
];
|
||
hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1";
|
||
};
|
||
users.groups.nasp = {};
|
||
## desktop
|
||
services.xserver = {
|
||
enable = true;
|
||
displayManager.gdm.enable = true;
|
||
desktopManager.gnome.enable = true;
|
||
xkb.layout = "us";
|
||
};
|
||
}
|
||
(lib.mkIf (config.networking.hostId != null) {
|
||
boot.supportedFilesystems = [ "zfs" ];
|
||
boot.zfs.forceImportRoot = false;
|
||
services.zfs.autoScrub.enable = true;
|
||
})
|
||
# nvidia
|
||
(lib.mkIf (nvidiaCfg.enable) {
|
||
nixpkgs.config.nvidia.acceptLicense = true;
|
||
hardware.nvidia = {
|
||
package = config.boot.kernelPackages.nvidiaPackages.legacy_470;
|
||
modesetting.enable = false;
|
||
powerManagement.enable = false;
|
||
powerManagement.finegrained = false;
|
||
open = false;
|
||
nvidiaSettings = true;
|
||
};
|
||
hardware.opengl = {
|
||
enable = true;
|
||
driSupport = true;
|
||
driSupport32Bit = true;
|
||
};
|
||
hardware.nvidia-container-toolkit.enable = true;
|
||
systemd.services.nvidia-container-toolkit-cdi-generator = {
|
||
path = [ pkgs.jq pkgs.moreutils ];
|
||
postStart = ''
|
||
jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json
|
||
'';
|
||
};
|
||
services.xserver.videoDrivers = [ "nvidia" ];
|
||
})
|
||
# docker
|
||
(lib.mkIf (dockerCfg.enable) {
|
||
virtualisation.docker = {
|
||
enable = true;
|
||
package = pkgs.docker_25;
|
||
daemon.settings = {
|
||
ipv6 = true;
|
||
fixed-cidr-v6 = "fddd:d0c1:1::/64";
|
||
experimental = true;
|
||
ip6tables = true;
|
||
live-restore = false;
|
||
};
|
||
};
|
||
environment.systemPackages = with pkgs; [
|
||
dnew
|
||
];
|
||
})
|
||
# nginx
|
||
(lib.mkIf (nginxCfg.enable) {
|
||
services.nginx = {
|
||
enable = true;
|
||
virtualHosts."default" = {
|
||
serverName = "_";
|
||
default = true;
|
||
locations."/" = {
|
||
return = "404";
|
||
};
|
||
};
|
||
};
|
||
})
|
||
(lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) {
|
||
services.nginx.virtualHosts."code-server" = {
|
||
serverName = "proxy.nasp.fit";
|
||
locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = {
|
||
extraConfig = ''
|
||
rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break;
|
||
proxy_pass "http://unix:/home2/run/$1.sock";
|
||
proxy_set_header Host $host;
|
||
proxy_set_header Upgrade $http_upgrade;
|
||
proxy_set_header Connection upgrade;
|
||
proxy_set_header Accept-Encoding gzip;
|
||
'';
|
||
};
|
||
};
|
||
})
|
||
# registry
|
||
(lib.mkIf (registryCfg.enable) {
|
||
systemd.timers."registry" = {
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "*:0/5:0";
|
||
Unit = "registry.service";
|
||
};
|
||
};
|
||
systemd.services."registry" = {
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
User = "root";
|
||
};
|
||
script = builtins.readFile ./scripts/registry.sh;
|
||
path = with pkgs; [ git bash su shadow getent ];
|
||
};
|
||
security.sudo.extraConfig = ''
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/docker
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/reboot
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/whoami
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/nvidia-smi
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/shutdown
|
||
%nasp ALL = (root) NOPASSWD: /run/current-system/sw/bin/ip
|
||
'';
|
||
})
|
||
# sops-nix
|
||
## gpg --fetch-keys "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa5d6250d1806caa8"
|
||
## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
|
||
## mkdir -p ~/.config/sops/age
|
||
## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt"
|
||
(lib.mkIf sopsCfg.enable {
|
||
sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml;
|
||
sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||
})
|
||
# telegraf
|
||
(lib.mkIf (telegrafCfg.enable) {
|
||
sops.secrets.telegraf = assert sopsCfg.enable; {};
|
||
services.telegraf = {
|
||
enable = true;
|
||
environmentFiles = [ "/run/secrets/telegraf" ];
|
||
extraConfig = {
|
||
agent = {
|
||
interval = "3s";
|
||
round_interval = true;
|
||
metric_batch_size = 1000;
|
||
metric_buffer_limit = 10000;
|
||
collection_jitter = "0s";
|
||
flush_interval = "30s";
|
||
flush_jitter = "3s";
|
||
precision = "0s";
|
||
hostname = assert (config.networking.hostName != ""); config.networking.hostName;
|
||
omit_hostname = false;
|
||
};
|
||
outputs.influxdb_v2 = {
|
||
urls = [ "\${INFLUX_URL}" ];
|
||
token = "\${INFLUX_TOKEN}";
|
||
organization = "nasp.fit";
|
||
bucket = "trash";
|
||
bucket_tag = "bucket";
|
||
exclude_bucket_tag = true;
|
||
timeout = "5s";
|
||
};
|
||
inputs.system = {
|
||
name_override = "system";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.cpu = {
|
||
percpu = false;
|
||
name_override = "cpu";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.kernel = {
|
||
name_override = "kernel";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.processes = {
|
||
name_override = "processes";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.mem = {
|
||
name_override = "memory";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "10s";
|
||
};
|
||
inputs.disk = {
|
||
ignore_fs = ["tmpfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs"];
|
||
name_override = "disk";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "30s";
|
||
};
|
||
inputs.diskio = {
|
||
name_override = "diskio";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.net = {
|
||
name_override = "net";
|
||
ignore_protocol_stats = true;
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
inputs.smart = {
|
||
path_smartctl = "/run/current-system/sw/bin/smartctl";
|
||
use_sudo = true;
|
||
name_override = "smartctl";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
interval = "30s";
|
||
};
|
||
inputs.temp = {
|
||
name_override = "temperture";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
};
|
||
};
|
||
security.sudo.extraConfig = ''
|
||
%telegraf ALL = (root) NOPASSWD: /run/current-system/sw/bin/smartctl
|
||
'';
|
||
})
|
||
(lib.mkIf (telegrafCfg.enable && nvidiaCfg.enable) {
|
||
services.telegraf.extraConfig = {
|
||
inputs.nvidia_smi = {
|
||
bin_path = "/run/current-system/sw/bin/nvidia-smi";
|
||
tags = { bucket = telegrafCfg.bucket; };
|
||
};
|
||
};
|
||
})
|
||
# g series
|
||
(lib.mkIf (gCfg.enable) {
|
||
## network
|
||
networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial);
|
||
networking.search = [ "nasp" ];
|
||
networking.nameservers = [ "192.168.16.118" ];
|
||
networking.extraHosts = ''
|
||
192.168.16.101 g1-nasp g1
|
||
192.168.16.102 g2-nasp g2
|
||
192.168.16.103 g3-nasp g3
|
||
192.168.16.104 g4-nasp g4
|
||
192.168.16.105 g5-nasp g5
|
||
192.168.16.106 g6-nasp g6
|
||
192.168.16.107 g7-nasp g7
|
||
192.168.16.108 g8-nasp g8
|
||
192.168.16.109 g9-nasp g9
|
||
192.168.16.110 g10-nasp g10
|
||
192.168.16.111 g11-nasp g11
|
||
192.168.16.112 g12-nasp g12
|
||
192.168.16.113 g13-nasp g13
|
||
192.168.16.114 g14-nasp g14
|
||
192.168.16.115 g15-nasp g15
|
||
192.168.16.116 g16-nasp g16
|
||
192.168.16.117 g17-nasp g17
|
||
192.168.16.118 g18-nasp g18 nasp.fit git.nasp.fit
|
||
192.168.16.119 g19-nasp g19
|
||
192.168.16.120 g20-nasp g20
|
||
'';
|
||
networking.firewall.extraCommands = ''
|
||
iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT
|
||
iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT
|
||
'';
|
||
fileSystems."/share" = {
|
||
device = "192.168.16.118:/data0/share";
|
||
fsType = "nfs";
|
||
};
|
||
## packages and services
|
||
nasp.docker.enable = lib.mkDefault true;
|
||
nasp.nvidia.enable = lib.mkDefault true;
|
||
nasp.registry.enable = lib.mkDefault true;
|
||
nasp.nginx.enable = lib.mkDefault true;
|
||
nasp.nginx.enableCodeServer = lib.mkDefault true;
|
||
nasp.sops.enable = lib.mkDefault true;
|
||
nasp.telegraf = {
|
||
enable = lib.mkDefault true;
|
||
bucket = "g-series";
|
||
};
|
||
services.ntp = {
|
||
enable = true;
|
||
servers = [ "192.168.16.118" ];
|
||
};
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth0Name != "") {
|
||
systemd.network.networks."10-eth0" = {
|
||
matchConfig.Name = gCfg.eth0Name;
|
||
networkConfig = {
|
||
DHCP = "no";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
address = [ "192.168.16.${builtins.toString (gCfg.serial + 100)}/24" ];
|
||
routes = [
|
||
{
|
||
routeConfig = {
|
||
Gateway = "192.168.16.118";
|
||
GatewayOnLink = true;
|
||
Metric = 90;
|
||
};
|
||
}
|
||
{
|
||
routeConfig = {
|
||
Gateway = "fd01:da8:bf:300::1";
|
||
GatewayOnLink = true;
|
||
Metric = 90;
|
||
};
|
||
}
|
||
];
|
||
};
|
||
networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true;
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth1Name != "") {
|
||
systemd.network.networks."10-eth1" = {
|
||
matchConfig.Name = gCfg.eth1Name;
|
||
networkConfig = {
|
||
DHCP = "yes";
|
||
IPv6AcceptRA = true;
|
||
};
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
})
|
||
(lib.mkIf (gCfg.enable && gCfg.eth2Name != "") {
|
||
systemd.network.networks."10-eth2" = {
|
||
matchConfig.Name = gCfg.eth2Name;
|
||
address = [ "12.12.12.${builtins.toString (gCfg.serial + 100)}/24" ];
|
||
linkConfig.RequiredForOnline = "no";
|
||
};
|
||
networking.rxe = {
|
||
enable = true;
|
||
interfaces = [ "${gCfg.eth2Name}" ];
|
||
};
|
||
})
|
||
]);
|
||
}
|