Skip to content

Commit

Permalink
Supports setting local environment vars (#167)
Browse files Browse the repository at this point in the history
* Supports setting local environment vars

Uses `.env` from current work dir to override or set new environment variables

* Refactor environment functions

Addresses code review suggestions and requests:
* Using t.TempDir
* Using cmp.Diff
* Sorting slices on unit test
* Returning errors on parsing environment variables
Also:
* Unexports some symbols that was only used by ./workspace

* Remove illed-thought word from comment

Thanks @zombiezen

Co-authored-by: Ross Light <[email protected]>

Co-authored-by: Ross Light <[email protected]>
  • Loading branch information
Ridai Govinda Pombo and zombiezen authored Sep 10, 2020
1 parent 3930bb0 commit 02bea15
Show file tree
Hide file tree
Showing 8 changed files with 223 additions and 85 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.0.3
github.com/golang/snappy v0.0.1 // indirect
github.com/google/go-cmp v0.5.0
github.com/google/go-cmp v0.5.2
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f // indirect
github.com/johnewart/archiver v3.1.4+incompatible
Expand Down Expand Up @@ -44,6 +44,7 @@ require (
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20200831141814-d751682dd103 // indirect
google.golang.org/grpc v1.31.1 // indirect
google.golang.org/protobuf v1.25.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down Expand Up @@ -580,6 +582,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
Expand Down
11 changes: 0 additions & 11 deletions plumbing/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,17 +126,6 @@ func DecompressBuffer(b *bytes.Buffer) error {
return nil
}

// Returns two empty strings and false if env isn't formed as "something=.*"
// Or else return env name, value and true
func SaneEnvironmentVar(env string) (name, value string, sane bool) {
s := strings.SplitN(env, "=", 2)
if sane = (len(s) == 2); sane {
name = s[0]
value = s[1]
}
return
}

func CloneRepository(remote GitRemote, inMem bool, basePath string) (rep *git.Repository, err error) {
if remote.Branch == "" {
return nil, fmt.Errorf("define a branch to clone repo %v", remote.Url)
Expand Down
60 changes: 9 additions & 51 deletions workspace/build_manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,8 @@ import (
"context"
"crypto/sha256"
"fmt"
"strings"
"text/template"

"github.com/joho/godotenv"
"github.com/yourbase/narwhal"
"github.com/yourbase/yb/plumbing"
"github.com/yourbase/yb/plumbing/log"
"github.com/yourbase/yb/runtime"
)

Expand Down Expand Up @@ -57,44 +52,22 @@ type ExecPhase struct {
BuildFirst []string `yaml:"build_first"`
}

func (e *ExecPhase) EnvironmentVariables(ctx context.Context, envName string, data runtime.RuntimeEnvironmentData) []string {

result := make([]string, 0)

for _, property := range e.Environment["default"] {

if _, _, ok := plumbing.SaneEnvironmentVar(property); ok {
interpolated, err := TemplateToString(property, data)
if err == nil {
result = append(result, interpolated)
} else {
result = append(result, property)
}
}
}
// environmentVariables returns a slice of parsed environment variables for this ExecPhase
func (e *ExecPhase) environmentVariables(ctx context.Context, envName string, data runtime.RuntimeEnvironmentData) ([]string, error) {
packs := make([][]string, 0)
packs = append(packs, e.Environment["default"])

fromContainers := make([]string, 0)
for k, v := range data.Containers.Environment(ctx) {
result = append(result, k, v)
fromContainers = append(fromContainers, k+"="+v)
}
packs = append(packs, fromContainers)

if envName != "default" {
for _, property := range e.Environment[envName] {
if _, _, ok := plumbing.SaneEnvironmentVar(property); ok {
result = append(result, property)
}
}
packs = append(packs, e.Environment[envName])
}

// Check for local .env file
err := godotenv.Load()
if err == nil {
localEnv, _ := godotenv.Read()
for k, v := range localEnv {
result = append(result, strings.Join([]string{k, v}, "="))
}
}

return result
return parseEnvironment(ctx, ".env", data, packs...)
}

type ExecDependencies struct {
Expand Down Expand Up @@ -185,18 +158,3 @@ func (b BuildManifest) BuildTargetList() []string {

return targets
}

func TemplateToString(templateText string, data interface{}) (string, error) {
t, err := template.New("generic").Parse(templateText)
if err != nil {
return "", err
}
var tpl bytes.Buffer
if err := t.Execute(&tpl, data); err != nil {
log.Errorf("Can't render template:: %v", err)
return "", err
}

result := tpl.String()
return result, nil
}
29 changes: 11 additions & 18 deletions workspace/build_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"time"

"github.com/yourbase/narwhal"
"github.com/yourbase/yb/plumbing"
"github.com/yourbase/yb/plumbing/log"
"github.com/yourbase/yb/runtime"
)
Expand Down Expand Up @@ -55,20 +54,9 @@ func (b BuildDependencies) ContainerList() []narwhal.ContainerDefinition {
return containers
}

func (bt BuildTarget) EnvironmentVariables(data runtime.RuntimeEnvironmentData) []string {
result := make([]string, 0)
for _, property := range bt.Environment {
if _, _, ok := plumbing.SaneEnvironmentVar(property); ok {
interpolated, err := TemplateToString(property, data)
if err == nil {
result = append(result, interpolated)
} else {
result = append(result, property)
}
}
}

return result
// environmentVariables returns a slice of parsed environment variables for this BuildTarget
func (bt BuildTarget) environmentVariables(data runtime.RuntimeEnvironmentData) ([]string, error) {
return parseEnvironment(context.TODO(), ".env", data, bt.Environment)
}

func (bt BuildTarget) Build(ctx context.Context, runtimeCtx *runtime.Runtime, output io.Writer, flags BuildFlags, packagePath string, globalDeps []string) ([]CommandTimer, error) {
Expand Down Expand Up @@ -206,17 +194,22 @@ func (bt BuildTarget) Build(ctx context.Context, runtimeCtx *runtime.Runtime, ou
}
}

envVars, err := bt.environmentVariables(runtimeCtx.EnvironmentData())
if err != nil {
return nil, err
}

// Do this after the containers are up
for _, envString := range bt.EnvironmentVariables(runtimeCtx.EnvironmentData()) {
if n, v, ok := plumbing.SaneEnvironmentVar(envString); ok {
for _, envString := range envVars {
if n, v, ok := checkAndSplitEnvVar(envString); ok {
builder.SetEnv(n, v)
} else {
log.Warnf("'%s' doesn't look like an environment variable", envString)
}
}

// Merge global deps with build target deps
err := (&bt).mergeDeps(globalDeps)
err = (&bt).mergeDeps(globalDeps)
if err != nil {
return stepTimes, err
}
Expand Down
83 changes: 83 additions & 0 deletions workspace/environment.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package workspace

import (
"bytes"
"context"
"errors"
"fmt"
"os"
"strings"
"text/template"

"github.com/joho/godotenv"
"github.com/yourbase/yb/plumbing/log"
"github.com/yourbase/yb/runtime"
)

// checkAndSplitEnvVar returns two empty strings and false
// if env isn't formed as "something=.*"
func checkAndSplitEnvVar(env string) (name, value string, sane bool) {
s := strings.SplitN(env, "=", 2)
if sane = (len(s) == 2); sane {
name = s[0]
value = s[1]
}
return
}

// parseEnvironment checks and process an arbitrary number of
// environment vars lists, trying to not duplicate anything
func parseEnvironment(ctx context.Context, envPath string, runtimeData runtime.RuntimeEnvironmentData, envPacks ...[]string) ([]string, error) {
envMap := make(map[string]string)
for _, envPack := range envPacks {
for _, prop := range envPack {
interpolated, err := templateToString(prop, runtimeData)
if err == nil {
prop = interpolated
} else {
return nil, err
}
if key, value, ok := checkAndSplitEnvVar(prop); ok {
envMap[key] = value
} else {
return nil, errors.New("invalid enviroment var spec: " + prop)
}
}
}

// Check and load .env file
localEnv, err := godotenv.Read(envPath)
if err == nil {
for k, v := range localEnv {
envMap[k] = v
}
} else {
log.Debugf("Dotenv load error: %v", err)
if !os.IsNotExist(err) {
return nil, fmt.Errorf("processing local .env: %w", err)
}
}

result := make([]string, 0)
for k, v := range envMap {
result = append(result, k+"="+v)
}
return result, nil
}

// templateToString process data and apply passed in templateText
// returning an interpolated string
func templateToString(templateText string, data interface{}) (string, error) {
t, err := template.New("generic").Parse(templateText)
if err != nil {
return "", err
}
var tpl bytes.Buffer
if err := t.Execute(&tpl, data); err != nil {
log.Errorf("Can't render template:: %v", err)
return "", err
}

result := tpl.String()
return result, nil
}
100 changes: 100 additions & 0 deletions workspace/environment_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package workspace

import (
"context"
"io/ioutil"
"path/filepath"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/yourbase/yb/runtime"
)

func Test_parseEnvironment(t *testing.T) {
type args struct {
envPath string
runtimeData runtime.RuntimeEnvironmentData
envPacks [][]string
}
dummyRuntimeData := runtime.RuntimeEnvironmentData{Containers: runtime.ContainerData{}}
egContents := []byte(`YB_PRECIOUS_SEKRET_KEY=something
THERE=no
YB_GITHUB_APP_ID=0000`)
tempDir := t.TempDir()
dotEnvFilePath := filepath.Join(tempDir, ".env")
if err := ioutil.WriteFile(dotEnvFilePath, egContents, 0644); err != nil {
t.Fatalf("Unable to write env file %s: %v", dotEnvFilePath, err)
}
t.Logf("Created %s for testing .env", dotEnvFilePath)

tests := []struct {
name string
args args
want []string
wantErr bool
}{
{
name: "No dotenv",
args: args{
envPath: ".env",
runtimeData: dummyRuntimeData,
envPacks: [][]string{
{
"DATABASE_URL=test-ME",
"YB_GITHUB_APP_ID=38644",
"YB_APP_URL=http://localhost:3000",
},
{
"AWS_SOMETHING_SOME=agoegoejo+Ej185",
},
},
},
want: []string{
"AWS_SOMETHING_SOME=agoegoejo+Ej185",
"DATABASE_URL=test-ME",
"YB_APP_URL=http://localhost:3000",
"YB_GITHUB_APP_ID=38644",
},
},
{
name: "With a dotenv",
args: args{
envPath: dotEnvFilePath,
runtimeData: dummyRuntimeData,
envPacks: [][]string{
{
"DATABASE_URL=test-ME",
"YB_GITHUB_APP_ID=38644",
"YB_APP_URL=http://localhost:3000",
},
{
"AWS_SOMETHING_SOME=agoegoejo+Ej185",
},
},
},
want: []string{
"AWS_SOMETHING_SOME=agoegoejo+Ej185",
"DATABASE_URL=test-ME",
"THERE=no",
"YB_APP_URL=http://localhost:3000",
"YB_GITHUB_APP_ID=0000",
"YB_PRECIOUS_SEKRET_KEY=something",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseEnvironment(context.Background(), tt.args.envPath, tt.args.runtimeData, tt.args.envPacks...)
if (err != nil) != tt.wantErr {
t.Errorf("parseEnvironment() error = %v, wantErr %v", err, tt.wantErr)
return
}
if diff := cmp.Diff(got, tt.want, cmpopts.SortSlices(func(i, j string) bool {
return i < j
})); diff != "" {
t.Errorf("parseEnvironment(), diff: %v", diff)
}
})
}
}
Loading

0 comments on commit 02bea15

Please sign in to comment.