edgelesssys · burgerdev · Oct 18, 2024
@@ -0,0 +1,126 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Wed, 23 Oct 2024 10:50:46 +0200
+Subject: [PATCH] measure agent-config.toml into PCR 10
+
+The agent config is security critical and needs to be measurable. It
+would be nice to measure the daemon config, too, but it contains
+hard-to-predict networking configuration (such as the node IP).
+
+This commit removes the files not relevant for Contrast out of an
+abundance of caution.
+---
+ src/cloud-api-adaptor/go.mod                  |  5 +++-
+ src/cloud-api-adaptor/go.sum                  |  2 ++
+ .../pkg/userdata/provision.go                 | 30 ++++++++++++++++---
+ 3 files changed, 32 insertions(+), 5 deletions(-)
+
+diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod
+index bd419f65c99429b3fd20f850509fb0223d82c41d..011870a713eaee698e577e2b55bc05eed37f2104 100644
+--- a/src/cloud-api-adaptor/go.mod
++++ b/src/cloud-api-adaptor/go.mod
+@@ -1,6 +1,8 @@
+ module github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor
+
+-go 1.21
++go 1.22
++
++toolchain go1.23.2
+
+ require (
+ 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
+@@ -52,6 +54,7 @@ require (
+ 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
+ 	github.com/docker/docker v25.0.5+incompatible
+ 	github.com/golang-jwt/jwt/v5 v5.2.1
++	github.com/google/go-tpm v0.9.1
+ 	github.com/moby/sys/mountinfo v0.7.1
+ 	github.com/pelletier/go-toml/v2 v2.1.0
+ 	github.com/sirupsen/logrus v1.9.3
+diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum
+index 0dd05c2182c891213db3c6920c82702bea6e1f3f..1ffa1f8366a60de6743314fab67e2dfe9b73d266 100644
+--- a/src/cloud-api-adaptor/go.sum
++++ b/src/cloud-api-adaptor/go.sum
+@@ -322,6 +322,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
+ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
++github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM=
++github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
+ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go
+index 5c3b6caf560d4c2fc97d3bd5fc23f1aef78750b3..e74169939e3526acfda17228231ad3e27376aeef 100644
+--- a/src/cloud-api-adaptor/pkg/userdata/provision.go
++++ b/src/cloud-api-adaptor/pkg/userdata/provision.go
+@@ -2,6 +2,7 @@ package userdata
+
+ import (
+ 	"context"
++	"crypto/sha256"
+ 	"fmt"
+ 	"log"
+ 	"os"
+@@ -12,6 +13,8 @@ import (
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/aws"
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/azure"
+ 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/docker"
++	"github.com/google/go-tpm/legacy/tpm2"
++	"github.com/google/go-tpm/tpmutil"
+ 	"gopkg.in/yaml.v2"
+ )
+
+@@ -162,6 +165,7 @@ func findConfigEntry(path string, cc *CloudConfig) []byte {
+ type entry struct {
+ 	path     string
+ 	optional bool
++	pcrIndex *int
+ }
+
+ func (f *entry) writeFile(cc *CloudConfig) error {
+@@ -179,6 +183,10 @@ func (f *entry) writeFile(cc *CloudConfig) error {
+ 		return fmt.Errorf("failed to create directory: %w", err)
+ 	}
+
++	if f.pcrIndex != nil {
++		extendPCR(*f.pcrIndex, bytes)
++	}
++
+ 	err = os.WriteFile(f.path, bytes, 0644)
+ 	if err != nil {
+ 		return fmt.Errorf("failed to write file: %w", err)
+@@ -189,11 +197,8 @@ func (f *entry) writeFile(cc *CloudConfig) error {
+
+ func processCloudConfig(cfg *Config, cc *CloudConfig) error {
+ 	entries := []entry{
+-		{path: cfg.paths.agentConfig, optional: false},
++		{path: cfg.paths.agentConfig, optional: false, pcrIndex: toPtr(10)},
+ 		{path: cfg.paths.daemonConfig, optional: false},
+-		{path: cfg.paths.aaConfig, optional: true},
+-		{path: cfg.paths.cdhConfig, optional: true},
+-		{path: cfg.paths.authJson, optional: true},
+ 	}
+
+ 	for _, e := range entries {
+@@ -228,3 +233,20 @@ func ProvisionFiles(cfg *Config) error {
+
+ 	return nil
+ }
++
++func extendPCR(pcrIndex int, data []byte) error {
++	digest := sha256.Sum256(data)
++
++	handle, err := tpm2.OpenTPM()
++	if err != nil {
++		return fmt.Errorf("opening TPM device: %w", err)
++	}
++	if err := tpm2.PCRExtend(handle, tpmutil.Handle(pcrIndex), tpm2.AlgSHA256, digest[:], ""); err != nil {
++		return fmt.Errorf("extending PCR %d: %w", pcrIndex, err)
++	}
++	return nil
++}
++
++func toPtr[A any](a A) *A {
++	return &a
++}
@@ -0,0 +1,86 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Wed, 23 Oct 2024 10:50:46 +0200
+Subject: [PATCH] set policy digest in agent config
+
+This allows verifying SetPolicyRequests on the agent side.
+
+Note: this patch needs to be supported by an equivalent patch for the
+Kata agent.
+---
+ src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go |  2 +-
+ src/cloud-api-adaptor/pkg/agent/config.go        | 10 +++++++++-
+ src/cloud-api-adaptor/pkg/util/cloud.go          |  9 +++++++++
+ 3 files changed, 19 insertions(+), 2 deletions(-)
+
+diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
+index 5a3ab96697a2f83f5499572eac76942b383c5a82..0a83683a156b8459a5706f9df12150e5114f3792 100644
+--- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
++++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
+@@ -239,7 +239,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
+ 		logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath)
+ 	}
+
+-	agentConfig, err := agent.CreateConfigFile(authFilePath)
++	agentConfig, err := agent.CreateConfigFile(authFilePath, util.GetPolicyFromAnnotation(req.Annotations))
+ 	if err != nil {
+ 		return nil, fmt.Errorf("creating agent config: %w", err)
+ 	}
+diff --git a/src/cloud-api-adaptor/pkg/agent/config.go b/src/cloud-api-adaptor/pkg/agent/config.go
+index 58bcc83435d62eaafe7d5972df5772741141d31d..c0d5b58dfc8cc46bc2e7721b1d7436d194058ea7 100644
+--- a/src/cloud-api-adaptor/pkg/agent/config.go
++++ b/src/cloud-api-adaptor/pkg/agent/config.go
+@@ -1,6 +1,9 @@
+ package agent
+
+ import (
++	"crypto/sha256"
++	"encoding/hex"
++
+ 	"github.com/pelletier/go-toml/v2"
+ )
+
+@@ -13,10 +16,11 @@ const (
+ type agentConfig struct {
+ 	ServerAddr           string `toml:"server_addr"`
+ 	GuestComponentsProcs string `toml:"guest_components_procs"`
++	PolicySHA256Hex      string `toml:"policy_digest_sha256_hex,omitempty"`
+ 	ImageRegistryAuth    string `toml:"image_registry_auth,omitempty"`
+ }
+
+-func CreateConfigFile(authJsonPath string) (string, error) {
++func CreateConfigFile(authJsonPath string, policy []byte) (string, error) {
+ 	var imageRegistryAuth string
+ 	if authJsonPath != "" {
+ 		imageRegistryAuth = "file://" + authJsonPath
+@@ -27,6 +31,10 @@ func CreateConfigFile(authJsonPath string) (string, error) {
+ 		GuestComponentsProcs: GuestComponentsProcs,
+ 		ImageRegistryAuth:    imageRegistryAuth,
+ 	}
++	if policy != nil {
++		digest := sha256.Sum256(policy)
++		config.PolicySHA256Hex = hex.EncodeToString(digest[:])
++	}
+
+ 	bytes, err := toml.Marshal(config)
+ 	if err != nil {
+diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go
+index b2ba396af486aafd5030cba90a62037470f466d7..e31749538a04b24a367320fdcb60db7614156c71 100644
+--- a/src/cloud-api-adaptor/pkg/util/cloud.go
++++ b/src/cloud-api-adaptor/pkg/util/cloud.go
+@@ -69,6 +69,15 @@ func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64)
+ 	return vcpuInt, memoryInt
+ }
+
++func GetPolicyFromAnnotation(annotations map[string]string) []byte {
++	// The policy is already base64-decoded in this annotation map.
++	policy, ok := annotations[hypannotations.Policy]
++	if !ok {
++		return nil
++	}
++	return []byte(policy)
++}
++
+ // Method to check if a string exists in a slice
+ func Contains(slice []string, s string) bool {
+ 	for _, item := range slice {
@@ -40,10 +40,23 @@ buildGoModule rec {
     hash = "sha256-5tDG0sEiRAsb259lPui5ntR6DVHDdcXhb04UESJzHhE=";
   };
 
+  patches = [
+    # Make the process-user-data job measure the agent config file into PCR10 (which is otherwise
+    # unused), so that it can be verified in an attestation report.
+    # The CAA attestation story is not decided yet. This patch enables one possible solution for
+    # Contrast.
+    ./0001-measure-agent-config.toml-into-PCR-10.patch
+    # Forward the expected policy hash as part of the agent-config.toml via instance user-data.
+    # Not upstreamable, like the patch above.
+    ./0002-set-policy-digest-in-agent-config.patch
+  ];
+
+  patchFlags = [ "-p3" ];
+
   sourceRoot = "${src.name}/src/cloud-api-adaptor";
 
   proxyVendor = true;
-  vendorHash = "sha256-kqzi7jRF3tQ4/yLkJXfZly4EvVKFb400/WXlN0WjYm8=";
+  vendorHash = "sha256-6FWMh2G5yM0QnhpfLS+fRfP6bpPtuGCeCvCNutog3YU=";
 
   nativeBuildInputs = lib.optional withLibvirt pkg-config;
 

@@ -0,0 +1,92 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Thu, 24 Oct 2024 17:06:26 +0200
+Subject: [PATCH] agent: read policy hash from config
+
+---
+ src/agent/Cargo.lock    | 1 +
+ src/agent/Cargo.toml    | 1 +
+ src/agent/src/config.rs | 8 ++++++++
+ src/agent/src/policy.rs | 5 ++++-
+ 4 files changed, 14 insertions(+), 1 deletion(-)
+
+diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
+index eae6db554824b9de0d9694d583d0b05d610c5d9a..731123e5a33dbf60108f1fdf188be40d63f94085 100644
+--- a/src/agent/Cargo.lock
++++ b/src/agent/Cargo.lock
+@@ -2818,6 +2818,7 @@ dependencies = [
+  "const_format",
+  "derivative",
+  "futures",
++ "hex",
+  "image-rs",
+  "ipnetwork",
+  "kata-sys-util",
+diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml
+index d5b3db965fe75cbccc182825a4115bdc57a9705b..44612495a9a7891441ae1bded737edec718e79e1 100644
+--- a/src/agent/Cargo.toml
++++ b/src/agent/Cargo.toml
+@@ -89,6 +89,7 @@ regorus = { version = "0.1.4", default-features = false, features = [
+ sha2 = { version = "0.10.6", optional = true }
+ sev = { version = "2.0.2", default-features = false, features = ["snp"], optional = true }
+ vmm-sys-util = { version = "0.11.0", optional = true }
++hex = "0.4.3"
+
+ [dev-dependencies]
+ tempfile = "3.1.0"
+diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs
+index 0ae3a94fbdcf8fb8c462d27939207cd600228a73..37a32017a099d0a9526729c5e92528f218d05c47 100644
+--- a/src/agent/src/config.rs
++++ b/src/agent/src/config.rs
+@@ -131,6 +131,8 @@ pub struct AgentConfig {
+     pub image_policy_file: String,
+     #[cfg(feature = "agent-policy")]
+     pub policy_file: String,
++    #[cfg(feature = "agent-policy")]
++    pub policy_digest_sha256_hex: String,
+ }
+
+ #[derive(Debug, Deserialize)]
+@@ -160,6 +162,8 @@ pub struct AgentConfigBuilder {
+     pub image_policy_file: Option<String>,
+     #[cfg(feature = "agent-policy")]
+     pub policy_file: Option<String>,
++    #[cfg(feature = "agent-policy")]
++    pub policy_digest_sha256_hex: Option<String>,
+ }
+
+ macro_rules! config_override {
+@@ -235,6 +239,8 @@ impl Default for AgentConfig {
+             image_policy_file: String::from(""),
+             #[cfg(feature = "agent-policy")]
+             policy_file: String::from(""),
++            #[cfg(feature = "agent-policy")]
++            policy_digest_sha256_hex: String::from(""),
+         }
+     }
+ }
+@@ -287,6 +293,8 @@ impl FromStr for AgentConfig {
+
+         #[cfg(feature = "agent-policy")]
+         config_override!(agent_config_builder, agent_config, policy_file);
++        #[cfg(feature = "agent-policy")]
++        config_override!(agent_config_builder, agent_config, policy_digest_sha256_hex);
+         Ok(agent_config)
+     }
+ }
+diff --git a/src/agent/src/policy.rs b/src/agent/src/policy.rs
+index 2f1da9ecd0d0ee1be06218d5bc9e58cd93defa8c..840385fc324fd29749bbff0c9642e6925b70d429 100644
+--- a/src/agent/src/policy.rs
++++ b/src/agent/src/policy.rs
+@@ -198,7 +198,10 @@ impl AgentPolicy {
+ }
+
+ fn verify_policy_digest(policy: &str) -> Result<()> {
+-    if let Ok(expected_digest) = get_tdx_mrconfigid() {
++    if !AGENT_CONFIG.policy_digest_sha256_hex.is_empty() {
++        let expected_digest = hex::decode(&AGENT_CONFIG.policy_digest_sha256_hex)?;
++        verify_sha_256(policy, expected_digest.as_slice())
++    } else if let Ok(expected_digest) = get_tdx_mrconfigid() {
+         info!(sl!(), "policy: TDX MrConfigId ({:?})", expected_digest);
+
+         // The MrConfigId used with TDX is longer than the host-data field used
@@ -0,0 +1,21 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Markus Rudy <[email protected]>
+Date: Fri, 18 Oct 2024 09:58:47 +0200
+Subject: [PATCH] runtime: forward policy to remote hypervisor
+
+---
+ src/runtime/virtcontainers/remote.go | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/runtime/virtcontainers/remote.go b/src/runtime/virtcontainers/remote.go
+index 047f09fe8c2db38469440508996b8359cfc8fcba..e95763e44c9bc3c167bbafa615069ef1b0af41a2 100644
+--- a/src/runtime/virtcontainers/remote.go
++++ b/src/runtime/virtcontainers/remote.go
+@@ -81,6 +81,7 @@ func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Net
+ 	annotations[hypannotations.DefaultVCPUs] = strconv.FormatUint(uint64(hypervisorConfig.NumVCPUs()), 10)
+ 	annotations[hypannotations.DefaultMemory] = strconv.FormatUint(uint64(hypervisorConfig.MemorySize), 10)
+ 	annotations[hypannotations.Initdata] = hypervisorConfig.Initdata
++	annotations[hypannotations.Policy] = hypervisorConfig.AgentPolicy
+
+ 	req := &pb.CreateVMRequest{
+ 		Id:                   id,
@@ -93,6 +93,16 @@ buildGoModule rec {
       ./0014-kata-sys-util-remove-obsolete-cgroups-dependency.patch
       ./0015-kata-sys-util-move-json-parsing-to-protocols-crate.patch
       ./0016-protocols-only-build-RLimit-impls-on-Linux.patch
+
+      # A peer-pod VM does not have HOSTDATA or MRCONFIGID, so the expected policy hash needs to
+      # be configured differently. This patch adds a policy hash config field to the agent config,
+      # which is passed by the CAA and loaded from user-data.
+      # The upstream plan-of-record is the initdata proposal, which will eventually provide all
+      # podvm configuration in a measurable way. Unfortunately, this proposal has diverged between
+      # Kata and CAA, so we're implementing our own solution here.
+      ./0017-agent-read-policy-hash-from-config.patch
+      # This patch makes the remote hypervisor aware of the workload policy.
+      ./0018-runtime-forward-policy-to-remote-hypervisor.patch
     ];
   };