Skip to content

Commit

Permalink
Add configuration options for AWS S3 server access logging (#3006)
Browse files Browse the repository at this point in the history 
* Add configuration options for AWS S3 server access logging

* add tests

---------

Co-authored-by: Adrian Eib <[email protected]>
  • Loading branch information
@findmyname666 @adrianeib
findmyname666 and adrianeib authored Aug 9, 2024
1 parent 2f978ff commit e79d9f4
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 61 deletions.
5 changes: 5 additions & 0 deletions docs/_docs/04_reference/config-blocks-and-attributes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,12 @@ For the `s3` backend, the following additional properties are supported in the `
- `disable_aws_client_checksums`: When `true`, disable computing and checking checksums on the request and response,
such as the CRC32 check for DynamoDB. See [#1059](https://github.com/gruntwork-io/terragrunt/issues/1059) for issue where this is a useful workaround.
- `accesslogging_bucket_name`: (Optional) When provided as a valid `string`, create an S3 bucket with this name to store the access logs for the S3 bucket used to store OpenTofu/Terraform state. If not provided, or string is empty or invalid S3 bucket name, then server access logging for the S3 bucket storing the Opentofu/Terraform state will be disabled. **Note:** When access logging is enabled supported encryption for state bucket is only `AES256`. Reference: [S3 server access logging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enable-server-access-logging.html)
- `accesslogging_target_object_partition_date_source`: (Optional) When provided as a valid `string`, it configures the `PartitionDateSource` option. This option is part of the `TargetObjectKeyFormat` and `PartitionedPrefix` AWS configurations, allowing you to configure the log object key format for the access log files. Reference: [Logging requests with server access logging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html).
- `accesslogging_target_prefix`: (Optional) When provided as a valid `string`, set the `TargetPrefix` for the access log objects in the S3 bucket used to store Opentofu/Terraform state. If set to **empty**`string`, then `TargetPrefix` will be set to **empty** `string`. If attribute is not provided at all, then `TargetPrefix` will be set to **default** value `TFStateLogs/`. This attribute won't take effect if the `accesslogging_bucket_name` attribute is not present.
- `skip_accesslogging_bucket_acl`: When set to `true`, the S3 bucket where access logs are stored will not be configured with bucket ACL.
- `skip_accesslogging_bucket_enforced_tls`: When set to `true`, the S3 bucket where access logs are stored will not be configured with a bucket policy that enforces access to the bucket via a TLS connection.
- `skip_accesslogging_bucket_public_access_blocking`: When set to `true`, the S3 bucket where access logs are stored will not have public access blocking enabled.
- `skip_accesslogging_bucket_ssencryption`: When set to `true`, the S3 bucket where access logs are stored will not be configured with server-side encryption.
- `bucket_sse_algorithm`: (Optional) The algorithm to use for server side encryption of the state bucket. Defaults to `aws:kms`.
- `bucket_sse_kms_key_id`: (Optional) The KMS Key to use when the encryption algorithm is `aws:kms`. Defaults to the AWS Managed `aws/s3` key.
- `assume_role`: (Optional) A configuration `map` to use when assuming a role (starting with Terraform 1.6 for Terraform). Override top level arguments
Expand Down
140 changes: 94 additions & 46 deletions remote/remote_state_s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,23 +40,28 @@ const (
type ExtendedRemoteStateConfigS3 struct {
remoteStateConfigS3 RemoteStateConfigS3

S3BucketTags map[string]string `mapstructure:"s3_bucket_tags"`
DynamotableTags map[string]string `mapstructure:"dynamodb_table_tags"`
AccessLoggingBucketTags map[string]string `mapstructure:"accesslogging_bucket_tags"`
SkipCredentialsValidation bool `mapstructure:"skip_credentials_validation"`
SkipBucketVersioning bool `mapstructure:"skip_bucket_versioning"`
SkipBucketSSEncryption bool `mapstructure:"skip_bucket_ssencryption"`
SkipBucketAccessLogging bool `mapstructure:"skip_bucket_accesslogging"`
SkipBucketRootAccess bool `mapstructure:"skip_bucket_root_access"`
SkipBucketEnforcedTLS bool `mapstructure:"skip_bucket_enforced_tls"`
SkipBucketPublicAccessBlocking bool `mapstructure:"skip_bucket_public_access_blocking"`
DisableBucketUpdate bool `mapstructure:"disable_bucket_update"`
EnableLockTableSSEncryption bool `mapstructure:"enable_lock_table_ssencryption"`
DisableAWSClientChecksums bool `mapstructure:"disable_aws_client_checksums"`
AccessLoggingBucketName string `mapstructure:"accesslogging_bucket_name"`
AccessLoggingTargetPrefix string `mapstructure:"accesslogging_target_prefix"`
BucketSSEAlgorithm string `mapstructure:"bucket_sse_algorithm"`
BucketSSEKMSKeyID string `mapstructure:"bucket_sse_kms_key_id"`
S3BucketTags map[string]string `mapstructure:"s3_bucket_tags"`
DynamotableTags map[string]string `mapstructure:"dynamodb_table_tags"`
AccessLoggingBucketTags map[string]string `mapstructure:"accesslogging_bucket_tags"`
SkipCredentialsValidation bool `mapstructure:"skip_credentials_validation"`
SkipBucketVersioning bool `mapstructure:"skip_bucket_versioning"`
SkipBucketSSEncryption bool `mapstructure:"skip_bucket_ssencryption"`
SkipBucketAccessLogging bool `mapstructure:"skip_bucket_accesslogging"`
SkipBucketRootAccess bool `mapstructure:"skip_bucket_root_access"`
SkipBucketEnforcedTLS bool `mapstructure:"skip_bucket_enforced_tls"`
SkipBucketPublicAccessBlocking bool `mapstructure:"skip_bucket_public_access_blocking"`
DisableBucketUpdate bool `mapstructure:"disable_bucket_update"`
EnableLockTableSSEncryption bool `mapstructure:"enable_lock_table_ssencryption"`
DisableAWSClientChecksums bool `mapstructure:"disable_aws_client_checksums"`
AccessLoggingBucketName string `mapstructure:"accesslogging_bucket_name"`
AccessLoggingTargetObjectPartitionDateSource string `mapstructure:"accesslogging_target_object_partition_date_source"`
AccessLoggingTargetPrefix string `mapstructure:"accesslogging_target_prefix"`
SkipAccessLoggingBucketAcl bool `mapstructure:"skip_accesslogging_bucket_acl"`
SkipAccessLoggingBucketEnforcedTLS bool `mapstructure:"skip_accesslogging_bucket_enforced_tls"`
SkipAccessLoggingBucketPublicAccessBlocking bool `mapstructure:"skip_accesslogging_bucket_public_access_blocking"`
SkipAccessLoggingBucketSSEncryption bool `mapstructure:"skip_accesslogging_bucket_ssencryption"`
BucketSSEAlgorithm string `mapstructure:"bucket_sse_algorithm"`
BucketSSEKMSKeyID string `mapstructure:"bucket_sse_kms_key_id"`
}

// These are settings that can appear in the remote_state config that are ONLY used by Terragrunt and NOT forwarded
Expand All @@ -75,7 +80,12 @@ var terragruntOnlyConfigs = []string{
"enable_lock_table_ssencryption",
"disable_aws_client_checksums",
"accesslogging_bucket_name",
"accesslogging_target_object_partition_date_source",
"accesslogging_target_prefix",
"skip_accesslogging_bucket_acl",
"skip_accesslogging_bucket_enforced_tls",
"skip_accesslogging_bucket_public_access_blocking",
"skip_accesslogging_bucket_ssencryption",
"bucket_sse_algorithm",
"bucket_sse_kms_key_id",
}
Expand Down Expand Up @@ -121,6 +131,33 @@ func (c *ExtendedRemoteStateConfigS3) GetAwsSessionConfig() *aws_helper.AwsSessi
}
}

// Builds AWS S3 logging input struct from the configuration.
func (c *ExtendedRemoteStateConfigS3) createS3LoggingInput() s3.PutBucketLoggingInput {
loggingInput := s3.PutBucketLoggingInput{
Bucket: aws.String(c.remoteStateConfigS3.Bucket),
BucketLoggingStatus: &s3.BucketLoggingStatus{
LoggingEnabled: &s3.LoggingEnabled{
TargetBucket: aws.String(c.AccessLoggingBucketName),
},
},
}

if c.AccessLoggingTargetPrefix != "" {
loggingInput.BucketLoggingStatus.LoggingEnabled.TargetPrefix = aws.String(c.AccessLoggingTargetPrefix)

}

if c.AccessLoggingTargetObjectPartitionDateSource != "" {
loggingInput.BucketLoggingStatus.LoggingEnabled.TargetObjectKeyFormat = &s3.TargetObjectKeyFormat{
PartitionedPrefix: &s3.PartitionedPrefix{
PartitionDateSource: aws.String(c.AccessLoggingTargetObjectPartitionDateSource),
},
}
}

return loggingInput
}

// The DynamoDB lock table attribute used to be called "lock_table", but has since been renamed to "dynamodb_table", and
// the old attribute name deprecated. The old attribute name has been eventually removed from Terraform starting with
// release 0.13. To maintain backwards compatibility, we support both names.
Expand Down Expand Up @@ -584,22 +621,30 @@ func configureAccessLogBucket(terragruntOptions *options.TerragruntOptions, s3Cl
return errors.WithStackTrace(err)
}

if err := EnablePublicAccessBlockingForS3Bucket(s3Client, config.AccessLoggingBucketName, terragruntOptions); err != nil {
return errors.WithStackTrace(err)
if !config.SkipAccessLoggingBucketPublicAccessBlocking {
if err := EnablePublicAccessBlockingForS3Bucket(s3Client, config.AccessLoggingBucketName, terragruntOptions); err != nil {
terragruntOptions.Logger.Errorf("Could not enable public access blocking on %s\n%s", config.AccessLoggingBucketName, err.Error())
return errors.WithStackTrace(err)
}
}

if err := EnableAccessLoggingForS3BucketWide(s3Client, &config.remoteStateConfigS3, terragruntOptions, config.AccessLoggingBucketName, config.AccessLoggingTargetPrefix); err != nil {
if err := EnableAccessLoggingForS3BucketWide(s3Client, config, terragruntOptions); err != nil {
terragruntOptions.Logger.Errorf("Could not enable access logging on %s\n%s", config.remoteStateConfigS3.Bucket, err.Error())
return errors.WithStackTrace(err)
}

if !config.SkipBucketSSEncryption {
if !config.SkipAccessLoggingBucketSSEncryption {
if err := EnableSSEForS3BucketWide(s3Client, config.AccessLoggingBucketName, s3.ServerSideEncryptionAes256, config, terragruntOptions); err != nil {
terragruntOptions.Logger.Errorf("Could not enable encryption on %s\n%s", config.AccessLoggingBucketName, err.Error())
return errors.WithStackTrace(err)
}
}

if err := EnableEnforcedTLSAccesstoS3Bucket(s3Client, config.AccessLoggingBucketName, config, terragruntOptions); err != nil {
return errors.WithStackTrace(err)
if !config.SkipAccessLoggingBucketEnforcedTLS {
if err := EnableEnforcedTLSAccesstoS3Bucket(s3Client, config.AccessLoggingBucketName, config, terragruntOptions); err != nil {
terragruntOptions.Logger.Errorf("Could not enable TLS access on %s\n%s", config.AccessLoggingBucketName, err.Error())
return errors.WithStackTrace(err)
}
}
return nil
}
Expand Down Expand Up @@ -666,7 +711,7 @@ func checkIfS3BucketNeedsUpdate(s3Client *s3.S3, config *ExtendedRemoteStateConf
}

if !config.SkipBucketAccessLogging && config.AccessLoggingBucketName != "" {
enabled, err := checkIfAccessLoggingForS3Enabled(s3Client, &config.remoteStateConfigS3, terragruntOptions)
enabled, err := checkS3AccessLoggingConfiguration(s3Client, config, terragruntOptions)
if err != nil {
return false, configBucket, err
}
Expand Down Expand Up @@ -1254,46 +1299,49 @@ func checkIfSSEForS3Enabled(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3
return false, nil
}

// EnableAccessLoggingForS3BucketWide Enable bucket-wide Access Logging for the AWS S3 bucket specified in the given config
func EnableAccessLoggingForS3BucketWide(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions, logsBucket string, logsBucketPrefix string) error {
if err := configureBucketAccessLoggingAcl(s3Client, aws.String(logsBucket), terragruntOptions); err != nil {
return errors.WithStackTraceAndPrefix(err, "Error configuring bucket access logging ACL on S3 bucket %s", config.Bucket)
}

terragruntOptions.Logger.Debugf("Putting bucket logging on S3 bucket %s with TargetBucket %s and TargetPrefix %s", config.Bucket, logsBucket, logsBucketPrefix)
// Enable bucket-wide Access Logging for the AWS S3 bucket specified in the given config
func EnableAccessLoggingForS3BucketWide(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
bucket := config.remoteStateConfigS3.Bucket
logsBucket := config.AccessLoggingBucketName
logsBucketPrefix := config.AccessLoggingTargetPrefix

loggingInput := s3.PutBucketLoggingInput{
Bucket: aws.String(config.Bucket),
BucketLoggingStatus: &s3.BucketLoggingStatus{
LoggingEnabled: &s3.LoggingEnabled{
TargetBucket: aws.String(logsBucket),
TargetPrefix: aws.String(logsBucketPrefix),
},
},
if !config.SkipAccessLoggingBucketAcl {
if err := configureBucketAccessLoggingAcl(s3Client, aws.String(logsBucket), terragruntOptions); err != nil {
return errors.WithStackTraceAndPrefix(err, "Error configuring bucket access logging ACL on S3 bucket %s", config.remoteStateConfigS3.Bucket)
}
}

loggingInput := config.createS3LoggingInput()
terragruntOptions.Logger.Debugf("Putting bucket logging on S3 bucket %s with TargetBucket %s and TargetPrefix %s\n%s", bucket, logsBucket, logsBucketPrefix, loggingInput)

if _, err := s3Client.PutBucketLogging(&loggingInput); err != nil {
return errors.WithStackTraceAndPrefix(err, "Error enabling bucket-wide Access Logging on AWS S3 bucket %s", config.Bucket)
return errors.WithStackTraceAndPrefix(err, "Error enabling bucket-wide Access Logging on AWS S3 bucket %s", config.remoteStateConfigS3.Bucket)
}

terragruntOptions.Logger.Debugf("Enabled bucket-wide Access Logging on AWS S3 bucket %s", config.Bucket)
terragruntOptions.Logger.Debugf("Enabled bucket-wide Access Logging on AWS S3 bucket %s", bucket)
return nil
}

func checkIfAccessLoggingForS3Enabled(s3Client *s3.S3, config *RemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) (bool, error) {
terragruntOptions.Logger.Debugf("Checking if Access Logging is enabled for AWS S3 bucket %s", config.Bucket)
func checkS3AccessLoggingConfiguration(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) (bool, error) {
terragruntOptions.Logger.Debugf("Checking if Access Logging is enabled for AWS S3 bucket %s", config.remoteStateConfigS3.Bucket)

input := &s3.GetBucketLoggingInput{Bucket: aws.String(config.Bucket)}
input := &s3.GetBucketLoggingInput{Bucket: aws.String(config.remoteStateConfigS3.Bucket)}
output, err := s3Client.GetBucketLogging(input)
if err != nil {
terragruntOptions.Logger.Debugf("Error checking if Access Logging is enabled for AWS S3 bucket %s: %s", config.Bucket, err.Error())
return false, errors.WithStackTraceAndPrefix(err, "Error checking if Access Logging is enabled for AWS S3 bucket %s", config.Bucket)
terragruntOptions.Logger.Debugf("Error checking if Access Logging is enabled for AWS S3 bucket %s: %s", config.remoteStateConfigS3.Bucket, err.Error())
return false, errors.WithStackTraceAndPrefix(err, "Error checking if Access Logging is enabled for AWS S3 bucket %s", config.remoteStateConfigS3.Bucket)
}

if output.LoggingEnabled == nil {
return false, nil
}

loggingInput := config.createS3LoggingInput()

if !reflect.DeepEqual(output.LoggingEnabled, loggingInput.BucketLoggingStatus.LoggingEnabled) {
return false, nil
}

return true, nil
}

Expand Down
Loading

0 comments on commit e79d9f4

Please sign in to comment.