diff --git a/configurations/default.nix b/configurations/default.nix index 511bc12..25959c6 100644 --- a/configurations/default.nix +++ b/configurations/default.nix @@ -12,6 +12,14 @@ {nixpkgs.hostPlatform = "x86_64-linux";} ]; }; + vm-full = inputs.nixpkgs.lib.nixosSystem { + modules = [ + self.nixosModules.default + ./full.nix + ./vm.nix + {nixpkgs.hostPlatform = "x86_64-linux";} + ]; + }; }; perSystem = _: { apps = { @@ -19,13 +27,23 @@ type = "app"; program = "${self.nixosConfigurations.vm-preview.config.system.build.vm}/bin/run-nixos-vm"; }; + vm-full = { + type = "app"; + program = "${self.nixosConfigurations.vm-full.config.system.build.vm}/bin/run-nixos-vm"; + }; }; devshells.default.commands = [ { name = "vm-preview"; category = "examples"; command = "${self.nixosConfigurations.vm-preview.config.system.build.vm}/bin/run-nixos-vm"; - help = "Start vm with cardano services on the preview network and ports forwarded to host"; + help = "Start vm with cardano services on the preview network with ports forwarded to host"; + } + { + name = "vm-full"; + category = "examples"; + command = "${self.nixosConfigurations.vm-full.config.system.build.vm}/bin/run-nixos-vm"; + help = "Start vm with all supported cardano services, http proxy and monitoring, with ports forwarded to host"; } ]; }; diff --git a/configurations/full.nix b/configurations/full.nix new file mode 100644 index 0000000..2e5d8e0 --- /dev/null +++ b/configurations/full.nix @@ -0,0 +1,33 @@ +{ + cardano = { + network = "preview"; + node.enable = true; + ogmios.enable = true; + kupo.enable = true; + db-sync.enable = true; + oura.enable = true; + blockfrost.enable = true; + http.enable = true; + monitoring.enable = true; + # monitoring.hosts = [ "localhost" ]; + }; + + networking.firewall.allowedTCPPorts = [3000 9090]; + + virtualisation.memorySize = 8192; + + virtualisation.forwardPorts = [ + { + # prometheus + from = "host"; + host.port = 9090; + guest.port = 9090; + } + { + # grafana + from = "host"; + host.port = 3000; + guest.port = 3000; + } + ]; +} diff --git a/docs/default.nix b/docs/default.nix index c13552f..42817f4 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -3,9 +3,7 @@ inputs, self, ... -}: let - rootConfig = config; -in { +}: { imports = [ ./render.nix ]; @@ -17,82 +15,82 @@ in { sidebarOptions = [ { anchor = "cardano"; - modules = [rootConfig.flake.nixosModules.cardano]; + modules = [config.flake.nixosModules.cardano]; namespaces = ["cardano"]; } { anchor = "cardano.cli"; - modules = [rootConfig.flake.nixosModules.cli]; + modules = [config.flake.nixosModules.cli]; namespaces = ["cardano.cli"]; } { anchor = "cardano.node"; - modules = [rootConfig.flake.nixosModules.node]; + modules = [config.flake.nixosModules.node]; namespaces = ["cardano.node"]; } { anchor = "services.cardano-node"; - modules = [rootConfig.flake.nixosModules.node {services.cardano-node.environment = "mainnet";}]; + modules = [config.flake.nixosModules.node {services.cardano-node.environment = "mainnet";}]; namespaces = ["services.cardano-node"]; } { anchor = "cardano.ogmios"; - modules = [rootConfig.flake.nixosModules.ogmios]; + modules = [config.flake.nixosModules.ogmios]; namespaces = ["cardano.ogmios"]; } { anchor = "services.ogmios"; - modules = [rootConfig.flake.nixosModules.ogmios]; + modules = [config.flake.nixosModules.ogmios]; namespaces = ["services.ogmios"]; } { anchor = "cardano.kupo"; - modules = [rootConfig.flake.nixosModules.kupo]; + modules = [config.flake.nixosModules.kupo]; namespaces = ["cardano.kupo"]; } { anchor = "services.kupo"; - modules = [rootConfig.flake.nixosModules.kupo]; + modules = [config.flake.nixosModules.kupo]; namespaces = ["services.kupo"]; } { anchor = "cardano.db-sync"; - modules = [rootConfig.flake.nixosModules.db-sync]; + modules = [config.flake.nixosModules.db-sync]; namespaces = ["cardano.db-sync"]; } { anchor = "services.cardano-db-sync"; - modules = [(rootConfig.flake.nixosModules.db-sync // {config.services.cardano-db-sync.cluster = "mainnet";})]; + modules = [(config.flake.nixosModules.db-sync // {config.services.cardano-db-sync.cluster = "mainnet";})]; namespaces = ["services.cardano-db-sync"]; } { anchor = "cardano.http"; - modules = [rootConfig.flake.nixosModules.http]; + modules = [config.flake.nixosModules.http]; namespaces = ["cardano.http"]; } { anchor = "services.http-proxy"; - modules = [rootConfig.flake.nixosModules.http]; + modules = [config.flake.nixosModules.http]; namespaces = ["services.http-proxy"]; } { anchor = "cardano.blockfrost"; - modules = [rootConfig.flake.nixosModules.blockfrost]; + modules = [config.flake.nixosModules.blockfrost]; namespaces = ["cardano.blockfrost"]; } { anchor = "services.blockfrost"; - modules = [rootConfig.flake.nixosModules.blockfrost]; + modules = [config.flake.nixosModules.blockfrost]; namespaces = ["services.blockfrost"]; } { anchor = "cardano.oura"; - modules = [rootConfig.flake.nixosModules.oura]; + modules = [config.flake.nixosModules.oura]; namespaces = ["cardano.oura"]; } { anchor = "services.oura"; - modules = [rootConfig.flake.nixosModules.oura]; + modules = [config.flake.nixosModules.oura]; namespaces = ["services.oura"]; } ]; diff --git a/docs/getting-started/cloud.md b/docs/getting-started/cloud.md new file mode 100644 index 0000000..b72f9c4 --- /dev/null +++ b/docs/getting-started/cloud.md @@ -0,0 +1,70 @@ +## Deploy to the Cloud + +⚠ Caution: The templates have an empty root password set in `vm.nix` for convenience. Be sure to remove the `vm.nix` import and use public key authentication before deploying to the cloud. + +To deploy the network of nodes and proxy to cloud providers such as AWS, Google Cloud, DigitalOcean, or Hetzner, some additional setup is required that is out of scope for this project as it depends on the deployment workflow. Here is an overview: + +### Deploy Cloud Infrastructure + +Cloud resources need to be created, for example with an Infrastructure-as-Code tool such as AWS CloudFormation or [OpenTofu](https://opentofu.org/). + +#### Cloud Machines + +Virtual machines (AWS EC2, droplet, etc.) need to be created, one for each node and one for the proxy. Synchronizing the blockchain takes a long time so auto scaling is not viable without extra setup, eg. using [cardanow](https://github.com/mlabs-haskell/cardanow/) to load snapshots or using shared network storage. + +#### Networking + +Private networking needs to be set up between the nodes and proxy, either via the cloud provider's native support (AWS VPC, etc.) or VPN such as wireguard. + +The nodes should be configured to be reachable only via the private network on which the proxy resides. This can be as simple as disabling public IPs for these machines. More complex setups have several options to configure networking, such as the services' listen addresses (`services.ogmios.host`, and similar NixOS options), OS firewall (`networking.firewall.*`) and cloud firewall (AWS security groups etc.). + +#### DNS Records + +To make the proxy reachable via a web address from the browser, DNS records need to be added. This is also required for HTTPS. The opinionated default is a separate subdomain for each service, this can be overridden via nginx configuration. + +Example DNS records, where the proxy public IP is `12.34.56.78`: + +```text +my.example.com A 12.34.56.78 +ogmios.my.example.com A 12.34.56.78 +kupo.my.example.com A 12.34.56.78 +``` + +Alternatively, a wildcard record may be added for `*.my.example.com`. + +### Operating System Configuration and Deployment + +NixOS has to be installed on the cloud machines. If the cloud provider does not have NixOS images, this can be achieved starting from mainstream distros like Debian or Ubuntu using [nixos-infect](https://github.com/elitak/nixos-infect) from cloud-init user data. A better option is to generate cloud images, eg. using [nixos-generators]https://github.com/nix-community/nixos-generators) and start the cloud machines from those. + +To deploy operating system configuration via SSH, `services.openssh` needs to be configured and `users.users.root.openssh.authorizedKeys.keys` set. Deployment can be done via `nixos-rebuild --flake . --target-host HOST` or using a [deployment app](https://github.com/nix-community/awesome-nix?tab=readme-ov-file#deployment-tools) such as [colmena](https://github.com/zhaofengli/colmena), or integrated into an infrastructure tool like [terraform-nixos](https://github.com/nix-community/terraform-nixos). + +#### Node Adresses + +The proxy needs to know how to reach the nodes. This is configured in `services.http-proxy.servers`. In the NixOS test environment, nodes are reachable by hostname, because static `hosts` entries are added for them automatically. This is not available in the cloud. Several options exist: + +- Use IP addresses. This is the simplest solution but can be inconvenient if private IPs are automatically assigned. Example: + `services.http-proxy.servers = [ "10.0.0.1" "10.0.0.2" "10.0.0.3" ];` +- Add `networking.hosts` entries to the proxy, possibly from the output of other tools such as OpenTofu. +- Use `mdns` for local DNS lookups by hostname. +- Use a local DNS server such as [dnsmasq](https://dnsmasq.org/doc.html), optionally with DHCP. +- Use internal DNS provided by the cloud service (AWS Route53 private hosted zone, etc.). + +#### Domain Name + +Once DNS records are created for the proxy as above, the domain name needs to be configured. This is also required for HTTPS. + +```nix +services.http-proxy.domainName = "my.example.com"; +``` + +The server will now respond to HTTP requests with `Hosts` header set to `my.example.com`, as well as making the services available at `ogmios.my.example.com` etc. + +#### HTTPS + +To serve public web pages and APIs, it is necessary to protect data integrity and confidentiality during transmission, so [HTTPS](https://en.wikipedia.org/wiki/HTTPS) needs to be enabled on the proxy. Once DNS and domain names are configured as above, this is easily achieved with the following option: + +```nix +services.http-proxy.https.enable = true; +``` + +This will set up [Let's Encrypt ACME TLS certificates](https://letsencrypt.org/how-it-works/) on the proxy server, enable HTTPS in the nginx web server and redirect all HTTP traffic to HTTPS. diff --git a/docs/getting-started/load-balancer.md b/docs/getting-started/load-balancer.md index 1ef6de0..9ab1bf0 100644 --- a/docs/getting-started/load-balancer.md +++ b/docs/getting-started/load-balancer.md @@ -22,7 +22,7 @@ An easy way to get started is to use the [flake template](https://zero-to-nix.co ``` mkdir myproject cd myproject -nix flake init --template github:mlabs-haskell/cardano.nix +nix flake init --template github:mlabs-haskell/cardano.nix#cluster git init git add . ``` @@ -33,79 +33,10 @@ The template provides virtual machine configurations for three nodes and a load `nix run .#vms` -The services will be available on ports forwarded from localhost: ogmios at http://localhost:8001 and kupo at http://localhost:8002 . +The services will be available on ports forwarded from localhost: ogmios at http://localhost:8001 and kupo at http://localhost:8002 . Grafana is available at http://localhost:8008 . Press `Ctrl+C` to stop the machines. -## Deploy to the Cloud - -To deploy the network of nodes and proxy to cloud providers such as AWS, Google Cloud, DigitalOcean, or Hetzner, some additional setup is required that is out of scope for this project as it depends on the deployment workflow. Here is an overview: - -### Deploy Cloud Infrastructure - -Cloud resources need to be created, for example with an Infrastructure-as-Code tool such as AWS CloudFormation or [OpenTofu](https://opentofu.org/). - -#### Cloud Machines - -Virtual machines (AWS EC2, droplet, etc.) need to be created, one for each node and one for the proxy. Synchronizing the blockchain takes a long time so auto scaling is not viable without extra setup, eg. using [cardanow](https://github.com/mlabs-haskell/cardanow/) to load snapshots or using shared network storage. - -#### Networking - -Private networking needs to be set up between the nodes and proxy, either via the cloud provider's native support (AWS VPC, etc.) or VPN such as wireguard. - -The nodes should be configured to be reachable only via the private network on which the proxy resides. This can be as simple as disabling public IPs for these machines. More complex setups have several options to configure networking, such as the services' listen addresses (`services.ogmios.host`, and similar NixOS options), OS firewall (`networking.firewall.*`) and cloud firewall (AWS security groups etc.). - -#### DNS Records - -To make the proxy reachable via a web address from the browser, DNS records need to be added. This is also required for HTTPS. The opinionated default is a separate subdomain for each service, this can be overridden via nginx configuration. - -Example DNS records, where the proxy public IP is `12.34.56.78`: - -```text -my.example.com A 12.34.56.78 -ogmios.my.example.com A 12.34.56.78 -kupo.my.example.com A 12.34.56.78 -``` - -Alternatively, a wildcard record may be added for `*.my.example.com`. - -### Operating System Configuration and Deployment - -NixOS has to be installed on the cloud machines. If the cloud provider does not have NixOS images, this can be achieved starting from mainstream distros like Debian or Ubuntu using [nixos-infect](https://github.com/elitak/nixos-infect) from cloud-init user data. A better option is to generate cloud images, eg. using [nixos-generators]https://github.com/nix-community/nixos-generators) and start the cloud machines from those. - -To deploy operating system configuration via SSH, `services.openssh` needs to be configured and `users.users.root.openssh.authorizedKeys.keys` set. Deployment can be done via `nixos-rebuild --flake . --target-host HOST` or using a [deployment app](https://github.com/nix-community/awesome-nix?tab=readme-ov-file#deployment-tools) such as [colmena](https://github.com/zhaofengli/colmena), or integrated into an infrastructure tool like [terraform-nixos](https://github.com/nix-community/terraform-nixos). - -#### Node Adresses - -The proxy needs to know how to reach the nodes. This is configured in `services.http-proxy.servers`. In the NixOS test environment, nodes are reachable by hostname, because static `hosts` entries are added for them automatically. This is not available in the cloud. Several options exist: - -- Use IP addresses. This is the simplest solution but can be inconvenient if private IPs are automatically assigned. Example: - `services.http-proxy.servers = [ "10.0.0.1" "10.0.0.2" "10.0.0.3" ];` -- Add `networking.hosts` entries to the proxy, possibly from the output of other tools such as OpenTofu. -- Use `mdns` for local DNS lookups by hostname. -- Use a local DNS server such as [dnsmasq](https://dnsmasq.org/doc.html), optionally with DHCP. -- Use internal DNS provided by the cloud service (AWS Route53 private hosted zone, etc.). - -#### Domain Name - -Once DNS records are created for the proxy as above, the domain name needs to be configured. This is also required for HTTPS. - -```nix -services.http-proxy.domainName = "my.example.com"; -``` - -The server will now respond to HTTP requests with `Hosts` header set to `my.example.com`, as well as making the services available at `ogmios.my.example.com` etc. - -#### HTTPS - -To serve public web pages and APIs, it is necessary to protect data integrity and confidentiality during transmission, so [HTTPS](https://en.wikipedia.org/wiki/HTTPS) needs to be enabled on the proxy. Once DNS and domain names are configured as above, this is easily achieved with the following option: - -```nix -services.http-proxy.https.enable = true; -``` - -This will set up [Let's Encrypt ACME TLS certificates](https://letsencrypt.org/how-it-works/) on the proxy server, enable HTTPS in the nginx web server and redirect all HTTP traffic to HTTPS. - ## Further reading Check out the following documentation: diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 862b45c..2d20142 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -64,6 +64,7 @@ nav: - Run a VM: getting-started/vm.md - Deploy: getting-started/deploy.md - Load Balancer: getting-started/load-balancer.md + - Deploy to Cloud: getting-started/cloud.md - Development: - development/develop.md - development/contributing.md diff --git a/docs/render.nix b/docs/render.nix index a01f5c8..911bf87 100644 --- a/docs/render.nix +++ b/docs/render.nix @@ -124,14 +124,14 @@ in { system, ... }: let - inherit (pkgs) stdenv mkdocs python311Packages; + inherit (pkgs) stdenv mkdocs python312Packages; my-mkdocs = pkgs.runCommand "my-mkdocs" { buildInputs = [ mkdocs - python311Packages.mkdocs-material + python312Packages.mkdocs-material ]; } '' mkdir -p $out/bin diff --git a/flake.lock b/flake.lock index 458f236..f5bf18d 100644 --- a/flake.lock +++ b/flake.lock @@ -1716,11 +1716,11 @@ }, "nixpkgs_8": { "locked": { - "lastModified": 1714253743, - "narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=", + "lastModified": 1728492678, + "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994", + "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 7bd2235..2d84808 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,6 @@ blockfrost = { url = "github:blockfrost/blockfrost-backend-ryo/v2.1.0"; # compatible with cardano-db-sync 13.3.0.0 }; - oura = { url = "github:txpipe/oura/v1.9.1"; inputs.crane.follows = "crane"; @@ -87,9 +86,9 @@ path = ./templates/default; description = "Example flake using cardano.nix"; }; - load-balancer = { - path = ./templates/load-balancer; - description = "Example flake using cardano.nix with load balancer"; + cluster = { + path = ./templates/cluster; + description = "Example flake for deploying a cardano.nix cluster with multiple nodes, load balancer and monitoring"; }; }; systems = [ diff --git a/modules/db-sync.nix b/modules/db-sync.nix index 103a675..017bfb6 100644 --- a/modules/db-sync.nix +++ b/modules/db-sync.nix @@ -5,7 +5,7 @@ }: let cfg = config.cardano.db-sync; dbsync-cfg = config.services.cardano-db-sync; - inherit (lib) mkIf mkMerge mkEnableOption; + inherit (lib) mkEnableOption mkIf mkMerge mkOptionDefault; in { options.cardano.db-sync = { enable = @@ -29,8 +29,12 @@ in { postgres.enable = mkEnableOption "Run postgres and connect dbsync to it." // {default = true;}; }; - config = mkIf cfg.enable (mkMerge [ + config = mkMerge [ { + # fix missing option default + services.cardano-db-sync.cluster = mkOptionDefault config.cardano.network; + } + (mkIf cfg.enable { services.cardano-db-sync = { enable = true; environment = config.services.cardano-node.environments.${config.cardano.network}; @@ -74,14 +78,14 @@ in { LockPersonality = true; }; }; - } - (mkIf (config.cardano.node.enable or false) { + }) + (mkIf (cfg.enable && config.cardano.node.enable or false) { systemd.services.cardano-db-sync = { after = ["cardano-node-socket.service"]; requires = ["cardano-node-socket.service"]; }; }) - (mkIf cfg.postgres.enable { + (mkIf (cfg.enable && cfg.postgres.enable) { services.postgresql = { enable = true; # see warnings: this should be same as user name @@ -100,5 +104,5 @@ in { } ]; }) - ]); + ]; } diff --git a/modules/default.nix b/modules/default.nix index 0d1cb78..1d1a28e 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -58,6 +58,11 @@ ./services/oura.nix ]; }; + monitoring = { + imports = [ + ./monitoring.nix + ]; + }; # the default module imports all modules default = { imports = diff --git a/modules/http.nix b/modules/http.nix index c055b8b..720df62 100644 --- a/modules/http.nix +++ b/modules/http.nix @@ -4,7 +4,7 @@ ... }: let cfg = config.cardano.http; - inherit (lib) mkIf mkDefault mkEnableOption optional; + inherit (lib) mkIf mkDefault mkEnableOption; in { options.cardano.http = { enable = mkEnableOption '' @@ -19,8 +19,6 @@ in { ''; }; config = mkIf cfg.enable { - networking.firewall.allowedTCPPorts = [80] ++ optional config.services.http-proxy.https.enable 443; - services.http-proxy = { enable = true; servers = mkIf (config.cardano.ogmios.enable || config.cardano.kupo.enable) (mkDefault ["127.0.0.1"]); @@ -36,6 +34,11 @@ in { inherit (config.services.kupo) port; inherit (config.services.kupo.package) version; }; + grafana = { + port = 3000; + inherit (config.services.grafana.package) version; + servers = mkDefault []; + }; }; }; }; diff --git a/modules/monitoring.nix b/modules/monitoring.nix new file mode 100644 index 0000000..646e74e --- /dev/null +++ b/modules/monitoring.nix @@ -0,0 +1,169 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.cardano.monitoring; + inherit (lib) mkIf mkEnableOption mkMerge mkOption types; +in { + options.cardano.monitoring = { + enable = mkEnableOption '' + monitoring services Prometheus and Grafana + ''; + targets = mkOption { + type = with types; listOf string; + default = ["localhost"]; + description = '' + List of hosts to to scrape prometheus metrics from. + ''; + }; + exporters = { + enable = mkOption { + type = with types; bool; + default = cfg.enable; + description = '' + Enable Prometheus exporters for running services. + ''; + }; + ports = mkOption { + type = with types; listOf port; + default = [ + config.cardano.node.prometheusExporter.port + config.cardano.oura.prometheusExporter.port + config.services.blockfrost.settings.server.port + config.services.cardano-db-sync.explorerConfig.PrometheusPort + config.services.ogmios.port + config.services.prometheus.exporters.node.port + config.services.prometheus.exporters.nginx.port + config.services.prometheus.exporters.postgres.port + ]; + description = '' + List of ports where prometheus exporters are exposed. This can be used to open ports in the firewall. + ''; + }; + openFirewall = mkOption { + type = with types; bool; + default = cfg.exporters.enable; + description = '' + Open firewall ports for prometheus exporters. + ''; + }; + }; + }; + config = mkMerge [ + (mkIf cfg.enable { + services.prometheus = { + enable = true; + scrapeConfigs = [ + { + job_name = "blockfrost"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.blockfrost.settings.server.port}") cfg.targets;}]; + } + { + job_name = "db_sync"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.cardano-db-sync.explorerConfig.PrometheusPort}") cfg.targets;}]; + } + { + job_name = "cardano-node"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.cardano.node.prometheusExporter.port}") cfg.targets;}]; + } + { + job_name = "kupo"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.kupo.port}") cfg.targets;}]; + metrics_path = "/health"; + } + { + job_name = "nginx"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.prometheus.exporters.nginx.port}") cfg.targets;}]; + } + { + job_name = "node"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.prometheus.exporters.node.port}") cfg.targets;}]; + } + { + job_name = "ogmios"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.ogmios.port}") cfg.targets;}]; + } + { + job_name = "oura"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.cardano.oura.prometheusExporter.port}") cfg.targets;}]; + } + { + job_name = "postgres"; + static_configs = [{targets = map (target: "${target}:${builtins.toString config.services.prometheus.exporters.postgres.port}") cfg.targets;}]; + } + ]; + }; + + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "0.0.0.0"; + }; + }; + provision = { + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + uid = "local_prometheus"; + url = "http://localhost:${builtins.toString config.services.prometheus.port}"; + } + ]; + dashboards.settings.providers = [ + { + name = "Dashboards"; + options.path = "/etc/grafana-dashboards"; + } + ]; + }; + }; + + environment.etc = { + "grafana-dashboards/node.json" = { + user = "grafana"; + group = "grafana"; + source = pkgs.fetchurl { + url = "https://grafana.com/api/dashboards/1860/revisions/37/download"; + sha256 = "sha256-1DE1aaanRHHeCOMWDGdOS1wBXxOF84UXAjJzT5Ek6mM="; + }; + }; + "grafana-dashboards/nginx.json" = { + user = "grafana"; + group = "grafana"; + source = pkgs.fetchurl { + url = "https://grafana.com/api/dashboards/14900/revisions/2/download"; + sha256 = "sha256-9iOEwKdFxOyw2T7Non4k2yUwiajWpH3qgQTyJRrttwM="; + }; + }; + "grafana-dashboards/postgres.json" = { + user = "grafana"; + group = "grafana"; + source = pkgs.fetchurl { + url = "https://grafana.com/api/dashboards/9628/revisions/7/download"; + sha256 = "sha256-xkzDitnr168JVR7oPmaaOPYqdufICSmvVmilhScys3Y="; + }; + }; + }; + }) + (mkIf cfg.exporters.enable { + cardano.node.prometheusExporter.enable = true; + cardano.oura.prometheusExporter.enable = true; + + services.blockfrost = mkIf config.services.bockfrost.enable or false { + settings.server.prometheusMetrics = true; + }; + + services.prometheus.exporters = { + node.enable = true; + nginx.enable = config.services.nginx.enable; + postgres.enable = config.services.postgresql.enable; + }; + }) + (mkIf cfg.exporters.openFirewall { + networking.firewall.allowedTCPPorts = cfg.exporters.ports; + }) + ]; +} diff --git a/modules/node.nix b/modules/node.nix index 037516b..11a0a79 100644 --- a/modules/node.nix +++ b/modules/node.nix @@ -5,6 +5,7 @@ ... }: let cfg = config.cardano.node; + inherit (builtins) elemAt match replaceStrings readFile; in { options.cardano.node = { enable = @@ -20,12 +21,27 @@ in { configPath = lib.mkOption { description = "Path to cardano-node configuration."; type = lib.types.path; - default = "${pkgs.cardano-configurations}/network/${config.cardano.network}/cardano-node/config.json"; - defaultText = lib.literalExpression "\${pkgs.cardano-configurations}/network/\${config.cardano.network}/cardano-node/config.json"; + default = "/etc/cardano-node/config.json"; + }; + + prometheusExporter.enable = + lib.mkEnableOption "prometheus exporter"; + + prometheusExporter.port = lib.mkOption { + description = "Port where Prometheus exporter is exposed."; + type = lib.types.port; + default = 12798; }; }; config = lib.mkIf cfg.enable { + environment.etc."cardano-node/config.json" = { + # hack to get config file path + text = readFile (elemAt (match ".* --config ([^ ]+) .*" (replaceStrings ["\n"] [" "] config.services.cardano-node.script)) 0); + user = "cardano-node"; + group = "cardano-node"; + }; + environment.variables = { CARDANO_NODE_SOCKET_PATH = cfg.socketPath; }; @@ -35,9 +51,12 @@ in { package = lib.mkDefault pkgs.cardano-node; inherit (cfg) socketPath; - nodeConfigFile = cfg.configPath; environment = config.cardano.network; + extraNodeConfig = lib.mkIf cfg.prometheusExporter.enable { + hasPrometheus = ["0.0.0.0" config.cardano.node.prometheusExporter.port]; + }; + # Listen on all interfaces. hostAddr = lib.mkDefault "0.0.0.0"; diff --git a/modules/oura.nix b/modules/oura.nix index cb34a33..9f82ea7 100644 --- a/modules/oura.nix +++ b/modules/oura.nix @@ -9,11 +9,19 @@ in { enable = lib.mkEnableOption "Oura" // {default = config.cardano.enable or false;}; + integrate = - lib.mkEnableOption '' - connect oura to local cardano-node via N2C - '' + lib.mkEnableOption ''connect oura to local cardano-node via N2C'' // {default = config.cardano.node.enable or false;}; + + prometheusExporter.enable = + lib.mkEnableOption "prometheus exporter"; + + prometheusExporter.port = lib.mkOption { + description = "Port where Prometheus exporter is exposed."; + type = lib.types.port; + default = 9186; + }; }; config = lib.mkIf cfg.enable { @@ -25,6 +33,7 @@ in { address = ["Unix" config.cardano.node.socketPath]; magic = config.cardano.network; }; + metrics.address = lib.mkIf cfg.prometheusExporter.enable "0.0.0.0:${builtins.toString cfg.prometheusExporter.port}"; }; }; diff --git a/modules/services/http-proxy.nix b/modules/services/http-proxy.nix index 4fe5962..7bd0ba3 100644 --- a/modules/services/http-proxy.nix +++ b/modules/services/http-proxy.nix @@ -4,10 +4,15 @@ ... }: let cfg = config.services.http-proxy; - inherit (lib) types listToAttrs mkOption mkEnableOption mapAttrs mkIf optionalString; + inherit (lib) types listToAttrs mkOption mkEnableOption mapAttrs mkIf optional optionalString; in { options.services.http-proxy = { enable = mkEnableOption "HTTP reverse proxy, TLS endpoint and load balancer"; + openFirewall = mkOption { + description = "Open firewall for HTTP and HTTPS."; + type = types.bool; + default = true; + }; domainName = mkOption { description = "Domain name. For each service a virtualHost is configured as a subdomain."; type = types.str; @@ -79,7 +84,7 @@ in { return = "200 ${service.version}"; extraConfig = "add_header Content-Type text/plain;"; }; - "/" = mkIf (service.port != null) { + "/" = mkIf (service.servers != [] && service.port != null) { proxyWebsockets = true; proxyPass = "http://${service.name}"; }; @@ -96,6 +101,8 @@ in { }; }; config = mkIf cfg.enable { + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ([80] ++ optional cfg.https.enable 443); + services.nginx = { enable = true; recommendedGzipSettings = true; @@ -106,7 +113,7 @@ in { statusPage = true; - upstreams = mapAttrs (_: cfg._mkUpstream) (lib.filterAttrs (_: s: s.port != null) cfg.services); + upstreams = mapAttrs (_: cfg._mkUpstream) (lib.filterAttrs (_: s: s.servers != [] && s.port != null) cfg.services); virtualHosts = mapAttrs (_: cfg._mkVirtualHost) cfg.services; }; }; diff --git a/templates/load-balancer/.gitignore b/templates/cluster/.gitignore similarity index 100% rename from templates/load-balancer/.gitignore rename to templates/cluster/.gitignore diff --git a/templates/load-balancer/flake.nix b/templates/cluster/flake.nix similarity index 54% rename from templates/load-balancer/flake.nix rename to templates/cluster/flake.nix index 8a61ce4..feffdbd 100644 --- a/templates/load-balancer/flake.nix +++ b/templates/cluster/flake.nix @@ -9,27 +9,38 @@ nixpkgs, ... }: { - nixosConfigurations = let - nixosSystem = modules: - nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = modules ++ [cardano-nix.nixosModules.default]; - }; - in { - node1 = nixosSystem [./preview.nix {networking.hostName = "node1";}]; - node2 = nixosSystem [./preview.nix {networking.hostName = "node2";}]; - node3 = nixosSystem [./preview.nix {networking.hostName = "node3";}]; - proxy = nixosSystem [./proxy.nix]; + nixosConfigurations = { + node1 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [{networking.hostName = "node1";} ./preview.nix cardano-nix.nixosModules.default]; + }; + node2 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [{networking.hostName = "node2";} ./preview.nix cardano-nix.nixosModules.default]; + }; + node3 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [{networking.hostName = "node3";} ./preview.nix cardano-nix.nixosModules.default]; + }; + status = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [./status.nix cardano-nix.nixosModules.default]; + }; + proxy = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [./proxy.nix cardano-nix.nixosModules.default]; + }; }; packages.x86_64-linux = { vms = ((import (nixpkgs.outPath + "/nixos/lib") {}).runTest { - name = "load-balancer"; + name = "cluster"; imports = [ { nodes.node1 = ./preview.nix; nodes.node2 = ./preview.nix; nodes.node3 = ./preview.nix; + nodes.status = ./status.nix; nodes.proxy = ./proxy.nix; testScript = _: '' start_all() diff --git a/templates/load-balancer/preview.nix b/templates/cluster/preview.nix similarity index 100% rename from templates/load-balancer/preview.nix rename to templates/cluster/preview.nix diff --git a/templates/load-balancer/proxy.nix b/templates/cluster/proxy.nix similarity index 79% rename from templates/load-balancer/proxy.nix rename to templates/cluster/proxy.nix index 2428b41..1ec5792 100644 --- a/templates/load-balancer/proxy.nix +++ b/templates/cluster/proxy.nix @@ -21,10 +21,15 @@ "node2" "node3" ]; + + services.http-proxy.services.grafana.servers = ["status"]; }; + # Enable Prometheus exporters and open firewall. Make sure not to expose these ports publicly when running in the cloud. + cardano.monitoring.exporters.enable = true; + # Configure services on separate ports, for easier forwarding from VM. Remove this if DNS is configured. - networking.firewall.allowedTCPPorts = [81 82]; + networking.firewall.allowedTCPPorts = [81 82 88]; services.nginx.virtualHosts.ogmios.listen = [ { addr = "0.0.0.0"; @@ -37,6 +42,12 @@ port = 82; } ]; + services.nginx.virtualHosts.grafana.listen = [ + { + addr = "0.0.0.0"; + port = 88; + } + ]; # Forward virtual machine port to host. Remove this if running on cloud. virtualisation.forwardPorts = [ @@ -55,6 +66,11 @@ host.port = 8002; guest.port = 82; } + { + from = "host"; + host.port = 8008; + guest.port = 88; + } { from = "host"; host.port = 2222; diff --git a/templates/cluster/status.nix b/templates/cluster/status.nix new file mode 100644 index 0000000..eb7c206 --- /dev/null +++ b/templates/cluster/status.nix @@ -0,0 +1,9 @@ +{ + cardano.monitoring = { + enable = true; + targets = ["proxy" "status" "node1" "node2" "node3"]; + }; + + # Grafana listen address. Do not expose this service publicly when running in the cloud, instead use the load balancer to proxy with HTTPS. + services.grafana.settings.server.http_addr = "0.0.0.0"; +} diff --git a/templates/load-balancer/vm.nix b/templates/cluster/vm.nix similarity index 100% rename from templates/load-balancer/vm.nix rename to templates/cluster/vm.nix diff --git a/templates/default/vm.nix b/templates/default/vm.nix index f064dd4..a0f42eb 100644 --- a/templates/default/vm.nix +++ b/templates/default/vm.nix @@ -1,7 +1,18 @@ -{modulesPath, ...}: { +{ + lib, + modulesPath, + ... +}: { imports = [ - (modulesPath + "/virtualisation/qemu-vm.nix") - (modulesPath + "/profiles/qemu-guest.nix") + "${modulesPath}/virtualisation/qemu-vm.nix" + "${modulesPath}/profiles/qemu-guest.nix" + ]; + + swapDevices = [ + { + device = "/swapfile"; + size = 4 * 1024; + } ]; # WARNING: don't use this in production @@ -11,9 +22,15 @@ virtualisation = { cores = 2; - memorySize = 2048; - diskSize = 100 * 1024; + memorySize = lib.mkDefault 4096; + diskSize = lib.mkDefault (100 * 1024); forwardPorts = [ + { + # http + from = "host"; + host.port = 8080; + guest.port = 80; + } { # cardano-node from = "host"; diff --git a/tests/default.nix b/tests/default.nix index d8b3350..6397b85 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -9,5 +9,6 @@ ./db-sync.nix ./blockfrost.nix ./oura.nix + ./monitoring.nix ]; } diff --git a/tests/monitoring.nix b/tests/monitoring.nix new file mode 100644 index 0000000..2075ab0 --- /dev/null +++ b/tests/monitoring.nix @@ -0,0 +1,25 @@ +{ + perSystem.vmTests.tests.monitoring = { + impure = true; + module = { + nodes.machine = {pkgs, ...}: { + cardano = { + network = "preview"; + node.enable = true; + monitoring.enable = true; + }; + + environment.systemPackages = with pkgs; [jq bc]; + }; + + testScript = '' + machine.wait_for_unit("cardano-node") + machine.wait_for_unit("prometheus") + machine.wait_for_unit("grafana") + machine.wait_until_succeeds('curl --silent --fail http://127.0.0.1:12798/metrics') + machine.wait_until_succeeds('curl --silent --fail http://127.0.0.1:3000/explore?schemaVersion=1&panes=%7B%225tr%22:%7B%22datasource%22:%22local_prometheus%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22cardano_node_metrics_cardano_build_info%22,%22range%22:false,%22instant%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22local_prometheus%22%7D,%22editorMode%22:%22builder%22,%22legendFormat%22:%22__auto%22,%22useBackend%22:false,%22disableTextWrap%22:false,%22fullMetaSearch%22:false,%22includeNullMetadata%22:true,%22format%22:%22table%22,%22exemplar%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1') + print('\nVM Test Succeeded.') + ''; + }; + }; +}