Skip to content

Commit

Permalink
Terragrunt performance improvements (#3307)
Browse files Browse the repository at this point in the history
* Performance testing

* Usage of common cache

* Added partial parse cache

* HCL cache fetching

* Add cache deep copy

* Fixed run_cmd cache

* Run cmd cache

* Add context cache

* Add test for git cache

* Mod update

* Add tracking of cache hit/miss

* Updated parse key

* Dependency outputs

* Add depednency caching

* Dependency cleanup

* Cleanup

* Deps update

* Code simplification

* Performance improvements for dependency fetching

* Cleanup

* Updated cache names
  • Loading branch information
denis256 authored Aug 6, 2024
1 parent c89bba5 commit 02ad89f
Show file tree
Hide file tree
Showing 27 changed files with 307 additions and 128 deletions.
4 changes: 3 additions & 1 deletion cli/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,11 @@ func (app *App) RunContext(ctx context.Context, args []string) error {
}
}(ctx)

ctx = config.WithConfigValues(ctx)

// init engine if required
if engine.IsEngineEnabled() {
ctx = engine.ContextWithEngine(ctx)
ctx = engine.WithEngineValues(ctx)
}
defer func(ctx context.Context) {
if err := engine.Shutdown(ctx); err != nil {
Expand Down
6 changes: 3 additions & 3 deletions cli/commands/terraform/creds/providers/amazonsts/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden
return nil, nil
}

if cached, hit := credentialsCache.Get(iamRoleOpts.RoleARN); hit {
if cached, hit := credentialsCache.Get(ctx, iamRoleOpts.RoleARN); hit {
provider.terragruntOptions.Logger.Debugf("Using cached credentials for IAM role %s.", iamRoleOpts.RoleARN)
return cached, nil
}
Expand All @@ -56,10 +56,10 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden
},
}

credentialsCache.Put(iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second))
credentialsCache.Put(ctx, iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second))

return creds, nil
}

// credentialsCache is a cache of credentials.
var credentialsCache = cache.NewExpiringCache[*providers.Credentials]()
var credentialsCache = cache.NewExpiringCache[*providers.Credentials]("credentialsCache")
2 changes: 1 addition & 1 deletion cli/commands/terraform/version_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const versionParts = 3
// - TerraformVersion
// TODO: Look into a way to refactor this function to avoid the side effect.
func checkVersionConstraints(ctx context.Context, terragruntOptions *options.TerragruntOptions) error {
configContext := config.NewParsingContext(context.Background(), terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints)
configContext := config.NewParsingContext(ctx, terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints)

partialTerragruntConfig, err := config.PartialParseConfigFile(
configContext,
Expand Down
14 changes: 9 additions & 5 deletions config/cache_test.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
package config

import (
"context"
"testing"

"github.com/gruntwork-io/terragrunt/internal/cache"
"github.com/stretchr/testify/assert"
)

const testCacheName = "TerragruntConfig"

func TestTerragruntConfigCacheCreation(t *testing.T) {
t.Parallel()

cache := cache.NewCache[TerragruntConfig]()
cache := cache.NewCache[TerragruntConfig](testCacheName)

assert.NotNil(t, cache.Mutex)
assert.NotNil(t, cache.Cache)
Expand All @@ -23,9 +26,10 @@ func TestTerragruntConfigCacheOperation(t *testing.T) {

testCacheKey := "super-safe-cache-key"

cache := cache.NewCache[TerragruntConfig]()
ctx := context.Background()
cache := cache.NewCache[TerragruntConfig](testCacheName)

actualResult, found := cache.Get(testCacheKey)
actualResult, found := cache.Get(ctx, testCacheKey)

assert.False(t, found)
assert.Empty(t, actualResult)
Expand All @@ -34,8 +38,8 @@ func TestTerragruntConfigCacheOperation(t *testing.T) {
IsPartial: true, // Any random property will be sufficient
}

cache.Put(testCacheKey, stubTerragruntConfig)
actualResult, found = cache.Get(testCacheKey)
cache.Put(ctx, testCacheKey, stubTerragruntConfig)
actualResult, found = cache.Get(ctx, testCacheKey)

assert.True(t, found)
assert.NotEmpty(t, actualResult)
Expand Down
21 changes: 9 additions & 12 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,11 @@ import (
const (
DefaultTerragruntConfigPath = "terragrunt.hcl"
DefaultTerragruntJsonConfigPath = "terragrunt.hcl.json"
FoundInFile = "found_in_file"

DefaultEngineType = "rpc"
)

const FoundInFile = "found_in_file"
iamRoleCacheName = "iamRoleCache"

const (
DefaultEngineType = "rpc"
MetadataTerraform = "terraform"
MetadataTerraformBinary = "terraform_binary"
MetadataTerraformVersionConstraint = "terraform_version_constraint"
Expand Down Expand Up @@ -696,12 +694,11 @@ func ReadTerragruntConfig(ctx context.Context, terragruntOptions *options.Terrag
return ParseConfigFile(parcingCtx, terragruntOptions.TerragruntConfigPath, nil)
}

var hclCache = cache.NewCache[*hclparse.File]()

// Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config
// included in some other config file when resolving relative paths.
func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
var config *TerragruntConfig
hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey)
err := telemetry.Telemetry(ctx, ctx.TerragruntOptions, "parse_config_file", map[string]interface{}{
"config_path": configPath,
"working_dir": ctx.TerragruntOptions.WorkingDir,
Expand All @@ -725,15 +722,15 @@ func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *I
}
var file *hclparse.File
var cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, ctx.TerragruntOptions.WorkingDir, dir, fileInfo.ModTime().UnixMicro())
if cacheConfig, found := hclCache.Get(cacheKey); found {
if cacheConfig, found := hclCache.Get(ctx, cacheKey); found {
file = cacheConfig
} else {
// Parse the HCL file into an AST body that can be decoded multiple times later without having to re-parse
file, err = hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
if err != nil {
return err
}
hclCache.Put(cacheKey, file)
hclCache.Put(ctx, cacheKey, file)
}
config, err = ParseConfig(ctx, file, includeFromChild)
if err != nil {
Expand Down Expand Up @@ -856,7 +853,7 @@ func ParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *Inc
}

// iamRoleCache - store for cached values of IAM roles
var iamRoleCache = cache.NewCache[options.IAMRoleOptions]()
var iamRoleCache = cache.NewCache[options.IAMRoleOptions](iamRoleCacheName)

// setIAMRole - extract IAM role details from Terragrunt flags block
func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) error {
Expand All @@ -866,14 +863,14 @@ func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *Incl
} else {
// as key is considered HCL code and include configuration
var key = fmt.Sprintf("%v-%v", file.Content(), includeFromChild)
var config, found = iamRoleCache.Get(key)
var config, found = iamRoleCache.Get(ctx, key)
if !found {
iamConfig, err := TerragruntConfigFromPartialConfig(ctx.WithDecodeList(TerragruntFlags), file, includeFromChild)
if err != nil {
return err
}
config = iamConfig.GetIAMRoleOptions()
iamRoleCache.Put(key, config)
iamRoleCache.Put(ctx, key, config)
}
// We merge the OriginalIAMRoleOptions into the one from the config, because the CLI passed IAMRoleOptions has
// precedence.
Expand Down
20 changes: 11 additions & 9 deletions config/config_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ const (
FuncNameEndsWith = "endswith"
FuncNameStrContains = "strcontains"
FuncNameTimeCmp = "timecmp"

sopsCacheName = "sopsCache"
)

// List of terraform commands that accept -lock-timeout
Expand Down Expand Up @@ -308,14 +310,14 @@ func parseGetEnvParameters(parameters []string) (EnvVar, error) {
return envVariable, nil
}

// runCommandCache - cache of evaluated `run_cmd` invocations
// see: https://github.com/gruntwork-io/terragrunt/issues/1427
var runCommandCache = cache.NewCache[string]()

// runCommand is a helper function that runs a command and returns the stdout as the interporation
// for each `run_cmd` in locals section, function is called twice
// result
func runCommand(ctx *ParsingContext, args []string) (string, error) {
// runCommandCache - cache of evaluated `run_cmd` invocations
// see: https://github.com/gruntwork-io/terragrunt/issues/1427
runCommandCache := cache.ContextCache[string](ctx, RunCmdCacheContextKey)

if len(args) == 0 {
return "", errors.WithStackTrace(EmptyStringNotAllowedError("parameter to the run_cmd function"))
}
Expand All @@ -341,7 +343,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) {
// To avoid re-run of the same run_cmd command, is used in memory cache for command results, with caching key path + arguments
// see: https://github.com/gruntwork-io/terragrunt/issues/1427
cacheKey := fmt.Sprintf("%v-%v", cachePath, args)
cachedValue, foundInCache := runCommandCache.Get(cacheKey)
cachedValue, foundInCache := runCommandCache.Get(ctx, cacheKey)
if foundInCache {
if suppressOutput {
ctx.TerragruntOptions.Logger.Debugf("run_cmd, cached output: [REDACTED]")
Expand All @@ -366,7 +368,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) {

// Persisting result in cache to avoid future re-evaluation
// see: https://github.com/gruntwork-io/terragrunt/issues/1427
runCommandCache.Put(cacheKey, value)
runCommandCache.Put(ctx, cacheKey, value)
return value, nil
}

Expand Down Expand Up @@ -728,7 +730,7 @@ func getModulePathFromSourceUrl(sourceUrl string) (string, error) {
//
// The cache keys are the canonical paths to the encrypted files, and the values are the
// plain-text result of the decrypt operation.
var sopsCache = cache.NewCache[string]()
var sopsCache = cache.NewCache[string](sopsCacheName)

// decrypts and returns sops encrypted utf-8 yaml or json data as a string
func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
Expand All @@ -751,7 +753,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
return "", errors.WithStackTrace(err)
}

if val, ok := sopsCache.Get(canonicalSourceFile); ok {
if val, ok := sopsCache.Get(ctx, canonicalSourceFile); ok {
return val, nil
}

Expand All @@ -762,7 +764,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {

if utf8.Valid(rawData) {
value := string(rawData)
sopsCache.Put(canonicalSourceFile, value)
sopsCache.Put(ctx, canonicalSourceFile, value)
return value, nil
}

Expand Down
32 changes: 25 additions & 7 deletions config/config_partial.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@ package config

import (
"fmt"
"os"
"path/filepath"

clone "github.com/huandu/go-clone"

"github.com/gruntwork-io/terragrunt/internal/cache"

"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"

"github.com/gruntwork-io/go-commons/errors"
"github.com/gruntwork-io/terragrunt/config/hclparse"
"github.com/gruntwork-io/terragrunt/internal/cache"
"github.com/gruntwork-io/terragrunt/util"
)

Expand Down Expand Up @@ -136,26 +140,39 @@ func DecodeBaseBlocks(ctx *ParsingContext, file *hclparse.File, includeFromChild
}

func PartialParseConfigFile(ctx *ParsingContext, configPath string, include *IncludeConfig) (*TerragruntConfig, error) {
file, err := hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey)

fileInfo, err := os.Stat(configPath)
if err != nil {
return nil, err
}
var file *hclparse.File
var cacheKey = fmt.Sprintf("configPath-%v-modTime-%v", configPath, fileInfo.ModTime().UnixMicro())

if cacheConfig, found := hclCache.Get(ctx, cacheKey); found {
file = cacheConfig
} else {
file, err = hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
if err != nil {
return nil, err
}
}

return TerragruntConfigFromPartialConfig(ctx, file, include)
}

var terragruntConfigCache = cache.NewCache[TerragruntConfig]()

// Wrapper of PartialParseConfigString which checks for cached configs.
// filename, configString, includeFromChild and decodeList are used for the cache key,
// by getting the default value (%#v) through fmt.
func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
var cacheKey = fmt.Sprintf("%#v-%#v-%#v-%#v", file.ConfigPath, file.Content(), includeFromChild, ctx.PartialParseDecodeList)

terragruntConfigCache := cache.ContextCache[*TerragruntConfig](ctx, RunCmdCacheContextKey)
if ctx.TerragruntOptions.UsePartialParseConfigCache {
if config, found := terragruntConfigCache.Get(cacheKey); found {
if config, found := terragruntConfigCache.Get(ctx, cacheKey); found {
ctx.TerragruntOptions.Logger.Debugf("Cache hit for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList)
return &config, nil
deepCopy := clone.Clone(config).(*TerragruntConfig)
return deepCopy, nil
}

ctx.TerragruntOptions.Logger.Debugf("Cache miss for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList)
Expand All @@ -167,7 +184,8 @@ func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File,
}

if ctx.TerragruntOptions.UsePartialParseConfigCache {
terragruntConfigCache.Put(cacheKey, *config)
putConfig := clone.Clone(config).(*TerragruntConfig)
terragruntConfigCache.Put(ctx, cacheKey, putConfig)
}

return config, nil
Expand Down
31 changes: 31 additions & 0 deletions config/context.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package config

import (
"context"

"github.com/gruntwork-io/terragrunt/config/hclparse"
"github.com/gruntwork-io/terragrunt/internal/cache"
)

type configKey byte

const (
HclCacheContextKey configKey = iota
TerragruntConfigCacheContextKey configKey = iota
RunCmdCacheContextKey configKey = iota
DependencyOutputCacheContextKey configKey = iota

hclCacheName = "hclCache"
configCacheName = "configCache"
runCmdCacheName = "runCmdCache"
dependencyOutputCacheName = "dependencyOutputCache"
)

// WithConfigValues add to context default values for configuration.
func WithConfigValues(ctx context.Context) context.Context {
ctx = context.WithValue(ctx, HclCacheContextKey, cache.NewCache[*hclparse.File](hclCacheName))
ctx = context.WithValue(ctx, TerragruntConfigCacheContextKey, cache.NewCache[*TerragruntConfig](configCacheName))
ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName))
ctx = context.WithValue(ctx, DependencyOutputCacheContextKey, cache.NewCache[*dependencyOutputCache](dependencyOutputCacheName))
return ctx
}
Loading

0 comments on commit 02ad89f

Please sign in to comment.