diff --git a/flake.nix b/flake.nix index 9541d4b..f8a1770 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,9 @@ "g18-next" "web-server" ]; + nasp = (import ./modules/nasp { + lib = nixpkgs.lib; + }); in { nixosConfigurations = { @@ -35,6 +38,7 @@ specialArgs = { inherit inputs; }; modules = [ ./hosts/${host}/configuration.nix + nasp.nixosModules.main sops-nix.nixosModules.sops ]; }; diff --git a/hosts/g1/configuration.nix b/hosts/g1/configuration.nix index 6a7c1b7..4a1bd16 100644 --- a/hosts/g1/configuration.nix +++ b/hosts/g1/configuration.nix @@ -4,7 +4,6 @@ imports = [ ./hardware-configuration.nix - ../modules/nasp.nix ]; nasp = { enable = true; diff --git a/hosts/web-server/configuration.nix b/hosts/web-server/configuration.nix deleted file mode 100644 index 1088819..0000000 --- a/hosts/web-server/configuration.nix +++ /dev/null @@ -1,81 +0,0 @@ -{ inputs, config, lib, pkgs, ... }: - -{ - imports = - [ - ./hardware-configuration.nix - ../modules/nasp.nix - ]; - - nasp = { - enable = true; - }; - - ## nix - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - ## hardware and system - boot.loader.systemd-boot.enable = true; - boot.loader.efi.canTouchEfiVariables = true; - time.hardwareClockInLocalTime = true; - ## network - services.resolved.enable = true; - networking.nameservers = [ - "166.111.8.29" - "166.111.8.28" - ]; - networking.networkmanager.enable = false; - networking.useDHCP = false; - systemd.network.enable = true; - networking.firewall.allowedTCPPorts = [ 80 443 12022 ]; - ## packages and services - nixpkgs.config.allowUnfree = true; - ## desktop - services.xserver = { - enable = true; - displayManager.gdm.enable = true; - desktopManager.gnome.enable = true; - xkb.layout = "us"; - }; - - # networking - networking.hostName = "web-server"; - systemd.network.networks."10-thunet" = { - matchConfig.Name = "enp3s0f0"; - networkConfig = { - DHCP = "no"; - IPv6AcceptRA = false; - }; - linkConfig = { - MACAddress = "58:97:bd:68:22:c2"; - }; - address = [ "166.111.68.109/28" ]; - routes = [ - { - routeConfig = { - Gateway = "166.111.68.97"; - GatewayOnLink = true; - Metric = 90; - }; - } - ]; - }; - - # service: nginx - services.nginx = { - enable = true; - virtualHosts."nasp.cs.tsinghua.edu.cn" = { - http2 = true; - addSSL = true; - enableACME = true; - locations."/" = { - root = "/data0/var/www/www"; - }; - }; - }; - security.acme = { - acceptTerms = true; - defaults.email = "xd21@mails.tsinghua.edu.cn"; - }; - - system.stateVersion = "24.05"; -} diff --git a/hosts/web-server/hardware-configuration.nix b/hosts/web-server/hardware-configuration.nix deleted file mode 100644 index 1716873..0000000 --- a/hosts/web-server/hardware-configuration.nix +++ /dev/null @@ -1,39 +0,0 @@ -# Do not modify this file! It was generated by ‘nixos-generate-config’ -# and may be overwritten by future invocations. Please make changes -# to /etc/nixos/configuration.nix instead. -{ config, lib, pkgs, modulesPath, ... }: - -{ - imports = - [ (modulesPath + "/installer/scan/not-detected.nix") - ]; - - boot.initrd.availableKernelModules = [ "ehci_pci" "ahci" "mpt3sas" "usb_storage" "usbhid" "sd_mod" ]; - boot.initrd.kernelModules = [ ]; - boot.kernelModules = [ "kvm-intel" ]; - boot.extraModulePackages = [ ]; - - fileSystems."/" = - { device = "/dev/disk/by-uuid/4bc05f99-244b-4af9-a751-4eb80199857c"; - fsType = "ext4"; - }; - - fileSystems."/boot" = - { device = "/dev/disk/by-uuid/75A7-1EBE"; - fsType = "vfat"; - options = [ "fmask=0022" "dmask=0022" ]; - }; - - swapDevices = [ ]; - - # Enables DHCP on each ethernet and wireless interface. In case of scripted networking - # (the default) this is the recommended approach. When using systemd-networkd it's - # still possible to use this option, but it's recommended to use it in conjunction - # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp3s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp3s0f1.useDHCP = lib.mkDefault true; - - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; - hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; -} diff --git a/modules/nasp/default.nix b/modules/nasp/default.nix new file mode 100644 index 0000000..7834867 --- /dev/null +++ b/modules/nasp/default.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + nixosModules.main = {config, pkgs, lib, ... }: (import ./main.nix { + inherit config pkgs lib; + }); +} \ No newline at end of file diff --git a/modules/nasp/flake.nix b/modules/nasp/flake.nix new file mode 100644 index 0000000..7252728 --- /dev/null +++ b/modules/nasp/flake.nix @@ -0,0 +1,12 @@ +{ + description = "Config Module for NASP's NixOS System"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/master"; + }; + outputs = { self, nixpkgs, ... }@inputs: rec { + nixosModules.main = {config, pkgs, lib, ... } : (import ./main.nix { + inherit config pkgs lib; + }); + + }; +} diff --git a/hosts/modules/nasp.nix b/modules/nasp/main.nix similarity index 72% rename from hosts/modules/nasp.nix rename to modules/nasp/main.nix index bffbd37..8b59ca8 100644 --- a/hosts/modules/nasp.nix +++ b/modules/nasp/main.nix @@ -9,7 +9,22 @@ let sopsCfg = mainCfg.sops; telegrafCfg = mainCfg.telegraf; gCfg = mainCfg.gSeries; + cCfg = mainCfg.cSeries; dnew = (pkgs.writeShellScriptBin "dnew" (builtins.readFile ./scripts/dnew)); + decToHex = + let + intToHex = [ + "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" + "a" "b" "c" "d" "e" "f" + ]; + toHex' = q: a: + if q > 0 + then (toHex' + (q / 16) + ((lib.elemAt intToHex (lib.mod q 16)) + a)) + else a; + in + v: toHex' v ""; in { options.nasp = { @@ -22,7 +37,6 @@ in }; nginx = { enable = lib.mkEnableOption "nginx web server"; - enableCodeServer = lib.mkEnableOption "proxy code server in docker"; }; registry = { enable = lib.mkEnableOption "the nasp registry"; @@ -41,7 +55,7 @@ in gSeries = { enable = lib.mkEnableOption "the g-series server configurations"; serial = lib.mkOption { - type = lib.types.int; + type = lib.types.ints.u8; description = "Serial of the machine (gX)"; }; eth0Name = lib.mkOption { @@ -60,6 +74,23 @@ in description = "Name of the RoCE NIC"; }; }; + cSeries = { + enable = lib.mkEnableOption "cpu server configurations"; + serial = lib.mkOption { + type = lib.types.ints.u8; + description = "Serial of the machine (cX)"; + }; + ethLanName = lib.mkOption { + type = lib.types.str; + default = ""; + description = "Name of the LAN NIC"; + }; + ethRDMAName = lib.mkOption { + type = lib.types.str; + default = ""; + description = "Name of the RDMA NIC"; + }; + }; }; # inplementation @@ -67,12 +98,14 @@ in # base { ## nix - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - nix.settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ]; + nix = { + settings.experimental-features = [ "nix-command" "flakes" ]; + settings.substituters = [ "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store" ]; + optimise.automatic = true; + }; ## hardware and system boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - time.hardwareClockInLocalTime = true; i18n.defaultLocale = "C.UTF-8"; i18n.extraLocaleSettings = lib.mkDefault { LC_ADDRESS = "zh_CN.UTF-8"; @@ -87,26 +120,56 @@ in }; time.timeZone = lib.mkDefault "Asia/Shanghai"; ## network + services.avahi.enable = true; + boot.kernel.sysctl = { + "net.core.default_qdisc" = "fq"; + "net.ipv4.tcp_congestion_control" = "bbr"; + }; services.resolved.enable = true; networking.networkmanager.enable = false; networking.useDHCP = false; systemd.network.enable = true; networking.firewall.allowedTCPPorts = [ 12022 ]; + networking.nftables.enable = true; + networking.extraHosts = '' + 192.168.16.1 ssh.nasp.fit git.nasp.fit jump.nasp.fit + 192.168.16.115 g15.nasp g15 lm1 + 192.168.16.116 g16.nasp g16 lm2 + 192.168.20.101 c1 c1.nasp + 192.168.20.102 c2 c2.nasp + ''; ## packages and services nixpkgs.config.allowUnfree = true; environment.systemPackages = with pkgs; [ - bash cmake curl file fzf gcc git gnumake htop nettools inetutils iproute2 iputils less man - openssh openssl python3 rdma-core sops sudo tmux util-linux vim wget zsh - # extended - acpi atop btop dialog dig dmidecode dos2unix ethtool fish gnupg iftop iotop killall lshw - lsof mtr netcat-gnu nethogs nmap pciutils plocate pstree pwgen ripgrep smartmontools socat - sysstat tcpdump unzip usbutils virt-what zip - # full + bash cmake curl dialog dig dmidecode e2fsprogs ethtool expect fd file fzf gcc git gnumake + gnupg htop iftop inetutils iotop iproute2 iputils jq less lrzsz lshw lsof man mtr + nettools nmap pciutils openssh openssl p7zip python3 ripgrep socat sops sudo tcpdump tmux + unzip usbutils util-linux vim wget zip zsh zssh zstd + acpi asciinema atop bat bridge-utils btop conntrack-tools dos2unix ffmpeg fish git-lfs + imagemagick iptstate killall libwebp ndisc6 netcat-gnu nethogs nix-diff nvme-cli pstree + pwgen smartmontools sysstat tldr virt-what wireshark zmap ]; programs.zsh.enable = true; programs.nix-ld.enable = true; + programs.git.lfs.enable = true; services.cron.enable = true; + services.locate = { + enable = true; + package = pkgs.plocate; + }; + fonts = { + enableDefaultPackages = true; + packages = with pkgs; [ + noto-fonts + noto-fonts-cjk-sans + noto-fonts-cjk-serif + noto-fonts-color-emoji + vista-fonts + vista-fonts-chs + roboto-mono + ]; + }; services.openssh.enable = true; services.openssh.settings.PermitRootLogin = "prohibit-password"; services.openssh.settings.PasswordAuthentication = false; @@ -116,6 +179,10 @@ in systemd.targets.suspend.enable = false; systemd.targets.hibernate.enable = false; systemd.targets.hybrid-sleep.enable = false; + fileSystems."/gshare" = { + device = "192.168.16.1:/data1/share"; + fsType = "nfs"; + }; ## users users.mutableUsers = true; users.users.root.openssh.authorizedKeys.keys = [ @@ -124,13 +191,6 @@ in "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyZILj+GxTUhdCgz2w1TxQ+aTcggnOJIb84qA4u271S asz258-17ac-bm-v0" ]; users.users.root.shell = pkgs.zsh; - system.activationScripts.dotfilesSetup.text = '' - if [ -d ~ -a ! -e ~/dotfiles/update.sh ]; then - source ${config.system.build.setEnvironment} - rm -rf ~/dotfiles - bash <(curl -fsSL dotfiles.cn) - fi - ''; users.users.nasp = { isNormalUser = true; createHome = true; @@ -139,6 +199,7 @@ in packages = with pkgs; [ firefox ]; + shell = pkgs.zsh; hashedPassword = "$y$j9T$Ei67I7VhQD6gF20/lNBUx0$jnrLqLNSJVCS959deKCamoOi4Q76nNeQ7/kDQCCABl1"; }; users.groups.nasp = {}; @@ -154,12 +215,12 @@ in boot.supportedFilesystems = [ "zfs" ]; boot.zfs.forceImportRoot = false; services.zfs.autoScrub.enable = true; + services.zfs.autoSnapshot.enable = true; }) # nvidia (lib.mkIf (nvidiaCfg.enable) { nixpkgs.config.nvidia.acceptLicense = true; hardware.nvidia = { - package = config.boot.kernelPackages.nvidiaPackages.legacy_470; modesetting.enable = false; powerManagement.enable = false; powerManagement.finegrained = false; @@ -171,12 +232,6 @@ in enable32Bit = true; }; hardware.nvidia-container-toolkit.enable = true; - systemd.services.nvidia-container-toolkit-cdi-generator = { - path = [ pkgs.jq pkgs.moreutils ]; - postStart = '' - jq '."containerEdits"."mounts" |= map(select(."containerPath" != "/usr/bin/nvidia-powerd"))' /run/cdi/nvidia-container-toolkit.json | sponge /run/cdi/nvidia-container-toolkit.json - ''; - }; services.xserver.videoDrivers = [ "nvidia" ]; }) # docker @@ -207,20 +262,19 @@ in return = "404"; }; }; - }; - }) - (lib.mkIf (nginxCfg.enable && nginxCfg.enableCodeServer) { - services.nginx.virtualHosts."code-server" = { - serverName = "proxy.nasp.fit"; - locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = { - extraConfig = '' - rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break; - proxy_pass "http://unix:/home2/run/$1.sock"; - proxy_set_header Host $host; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection upgrade; - proxy_set_header Accept-Encoding gzip; - ''; + virtualHosts."code-server" = { + serverName = "proxy.nasp.fit"; + locations."~ ^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" = { + proxyWebsockets = true; + extraConfig = '' + rewrite "^/${config.networking.hostName}/([A-Za-z0-9]+)/(.*)" /$2 break; + proxy_pass "http://unix:/home2/run/$1.sock"; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + proxy_set_header Accept-Encoding gzip; + ''; + }; }; }; }) @@ -250,10 +304,6 @@ in ## nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age' ## mkdir -p ~/.config/sops/age ## nix-shell -p ssh-to-age --run "ssh-to-age -private-key -i /etc/ssh/ssh_host_ed25519_key > ~/.config/sops/age/keys.txt" - (lib.mkIf sopsCfg.enable { - sops.defaultSopsFile = ../${config.networking.hostName}/secrets.yaml; - sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; - }) # telegraf (lib.mkIf (telegrafCfg.enable) { sops.secrets.telegraf = assert sopsCfg.enable; {}; @@ -349,46 +399,19 @@ in }) # g series (lib.mkIf (gCfg.enable) { + warnings = [ "To my knowledge, the g-series server is deprecated (2025-10-28)." ]; ## network networking.hostName = assert (gCfg.serial > 0); "g" + (builtins.toString gCfg.serial); - networking.search = [ "" ]; networking.nameservers = [ "192.168.16.1" ]; - networking.extraHosts = '' - 192.168.16.1 nasp.fit git.nasp.fit - 192.168.16.101 g1.nasp g1 - 192.168.16.102 g2.nasp g2 - 192.168.16.103 g3.nasp g3 - 192.168.16.104 g4.nasp g4 - 192.168.16.105 g5.nasp g5 - 192.168.16.106 g6.nasp g6 - 192.168.16.107 g7.nasp g7 - 192.168.16.108 g8.nasp g8 - 192.168.16.109 g9.nasp g9 - 192.168.16.110 g10.nasp g10 - 192.168.16.111 g11.nasp g11 - 192.168.16.112 g12.nasp g12 - 192.168.16.113 g13.nasp g13 - 192.168.16.114 g14.nasp g14 - 192.168.16.115 g15.nasp g15 - 192.168.16.116 g16.nasp g16 - 192.168.16.117 g17.nasp g17 - 192.168.16.118 g18.nasp g18 - 192.168.16.119 g19.nasp g19 + networking.firewall.extraInputRules = '' + ip saddr 192.168.16.0/24 accept + ip saddr 12.12.12.0/24 accept ''; - networking.firewall.extraCommands = '' - iptables -A INPUT -s 192.168.16.0/24 -j ACCEPT - iptables -A INPUT -s 12.12.12.0/24 -j ACCEPT - ''; - fileSystems."/gshare" = { - device = "192.168.16.1:/data1/share"; - fsType = "nfs"; - }; ## packages and services nasp.docker.enable = lib.mkDefault true; nasp.nvidia.enable = lib.mkDefault true; nasp.registry.enable = lib.mkDefault true; nasp.nginx.enable = lib.mkDefault true; - nasp.nginx.enableCodeServer = lib.mkDefault true; nasp.sops.enable = lib.mkDefault true; nasp.telegraf = { enable = lib.mkDefault true; @@ -413,11 +436,6 @@ in GatewayOnLink = true; Metric = 90; } - { - Gateway = "fd01:da8:bf:300::1"; - GatewayOnLink = true; - Metric = 90; - } ]; }; networking.interfaces.${gCfg.eth0Name}.wakeOnLan.enable = true; @@ -443,5 +461,46 @@ in interfaces = [ "${gCfg.eth2Name}" ]; }; }) + (lib.mkIf (cCfg.enable) { + networking.hostName = "c" + (builtins.toString cCfg.serial); + networking.nameservers = [ "192.168.20.1" ]; + networking.firewall.extraInputRules = '' + ip saddr 192.168.20.0/24 accept + ip6 saddr fd01:da8:bf:14::/64 accept + ''; + nasp.docker.enable = lib.mkDefault true; + nasp.registry.enable = lib.mkDefault true; + nasp.nginx.enable = lib.mkDefault true; + systemd.network.networks."10-eth-lan" = { + matchConfig.Name = cCfg.ethLanName; + networkConfig = { + DHCP = "no"; + IPv6AcceptRA = false; + }; + address = [ + "192.168.20.${builtins.toString (cCfg.serial + 100)}/24" + "fd01:da8:bf:14::${decToHex (cCfg.serial + 100)}/64" + ]; + routes = [ + { Gateway = "192.168.20.1"; } + { Gateway = "fd01:da8:bf:14::1"; } + ]; + }; + networking.interfaces.${cCfg.ethLanName}.wakeOnLan.enable = true; + }) + (lib.mkIf (cCfg.enable && cCfg.ethRDMAName != "") { + systemd.network.networks."10-eth-rdma" = { + matchConfig.Name = cCfg.ethRDMAName; + address = [ "12.12.12.${builtins.toString (cCfg.serial + 100)}/24" ]; + linkConfig.RequiredForOnline = "no"; + }; + networking.rxe = { + enable = true; + interfaces = [ "${cCfg.ethRDMAName}" ]; + }; + networking.firewall.extraInputRules = '' + ip saddr 12.12.12.0/24 accept + ''; + }) ]); } diff --git a/hosts/modules/scripts/dnew b/modules/nasp/scripts/dnew similarity index 100% rename from hosts/modules/scripts/dnew rename to modules/nasp/scripts/dnew diff --git a/hosts/modules/scripts/registry.sh b/modules/nasp/scripts/registry.sh similarity index 100% rename from hosts/modules/scripts/registry.sh rename to modules/nasp/scripts/registry.sh