Set locale environment variables in build environment

Cherry-pick of 2b87527 for 0.5

I selected C.UTF-8 as a somewhat neutral default that is likely to be
available in many environments. The user can always override it in their
target configuration.

Also corrected the non-POSIX-compliant `TZ` value from `UTC` to `UTC0`.

I added a small companion document to specify the build environment and
what guarantees we provide going forward.

[Fixes ch3824]

CHANGELOG.md

@@ -4,7 +4,25 @@ The format is based on [Keep a Changelog][], and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
-[Unreleased]: https://github.com/yourbase/yb/compare/v0.5.6...HEAD
+[Unreleased]: https://github.com/yourbase/yb/compare/v0.5.7...HEAD
+
+## [0.5.7][] - 2021-03-01
+
+Version 0.5.7 backports a fix for a locale environment variable issue.
+
+[0.5.7]: https://github.com/yourbase/yb/releases/tag/v0.5.7
+
+### Changed
+
+-  The build environment now sets `LANG` and other locale environment variables
+   to `C.UTF-8` or the closest approximation thereof. Previously, these
+   variables were unset, which caused problems with programs that required a
+   UTF-8 character set to function properly, like those written in Ruby or Python.
+
+### Fixed
+
+-  The `TZ` environment variable is now set to `UTC0` by default. Previously,
+   it was set to `UTC`, which is not a POSIX-conforming value.
 
 ## [0.5.6][] - 2021-02-11
 

internal/biome/README.md

@@ -0,0 +1,66 @@
+# Build Environment Reference
+
+**WIP specification. This information should eventually live in end-user
+reference documentation.**
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
+"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
+interpreted as described in [RFC 2119][].
+
+[RFC 2119]: https://tools.ietf.org/html/rfc2119
+
+## Environment Variables
+
+Each target **SHALL** run in a separate, isolated environment. For example,
+a target may run in a Docker container or a chroot jail. A target _MAY_ run on
+a different host from the build runner. At a minimum, the following environment
+variables **MUST** be set for commands running in POSIX environments:
+
+-  `HOME` **MUST** be set to the path of an readable and writable directory.
+   This _SHOULD NOT_ be the same as the user's actual `HOME` directory to
+   keep builds reproducible.
+-  `LOGNAME` and `USER` **MUST** be set to the name of the POSIX user running
+   the command (not the runner of `yb`).
+-  `PATH` **MUST NOT** be empty.
+-  `TZ` **MUST** be set to `UTC0`.
+-  `LANG` _SHOULD_ be set to `C.UTF-8` if the environment supports it.
+   Otherwise, `LANG` **MUST** be set to `C`.
+-  One of `LC_ALL` or `LC_CTYPE` **MUST** be set to C-like locale category
+   whose `charmap` is UTF-8. If `LC_CTYPE` is set, `LC_ALL` **MUST NOT** be set.
+
+### Examples of Locale Settings
+
+For Linux systems:
+
+```
+LANG=C.UTF-8
+LC_ALL=C.UTF-8
+```
+
+For macOS systems:
+
+```
+LANG=C
+LC_CTYPE=UTF-8
+```
+
+### Further Reading
+
+-  [POSIX.1-2017 Environment Variables](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html)
+   describes the meaning of the standard environment variables.
+-  [PEP 538](https://www.python.org/dev/peps/pep-0538/) documents Python's
+   process for bootstrapping a UTF-8 locale with rationale and platform-specific
+   caveats.
+
+## Expected Userspace
+
+The build environment that a target's commands run in **MUST** include the
+standard [POSIX utilities][]. yb also depends on the following utilities being
+available:
+
+-  `python` on non-Linux to fill in `readlink --canonicalize-existing` behavior
+-  `readlink` on Linux
+-  `tar`
+-  `unzip`
+
+[POSIX utilities]: https://pubs.opengroup.org/onlinepubs/9699919799/idx/utilities.html

internal/biome/biome.go

@@ -195,8 +195,8 @@ func (l Local) Run(ctx context.Context, invoke *Invocation) error {
 		"HOME=" + l.HomeDir,
 		"LOGNAME=" + os.Getenv("LOGNAME"),
 		"USER=" + os.Getenv("USER"),
-		"TZ=UTC",
 	}
+	c.Env = appendStandardEnv(c.Env, runtime.GOOS)
 	c.Env = invoke.Env.appendTo(c.Env, os.Getenv("PATH"), filepath.ListSeparator)
 	c.Dir = dir
 	c.Stdin = invoke.Stdin
@@ -208,6 +208,16 @@ func (l Local) Run(ctx context.Context, invoke *Invocation) error {
 	return nil
 }
 
+func appendStandardEnv(env []string, biomeOS string) []string {
+	env = append(env, "TZ=UTC0")
+	if biomeOS == MacOS {
+		env = append(env, "LANG=C", "LC_CTYPE=UTF-8")
+	} else {
+		env = append(env, "LANG=C.UTF-8", "LC_ALL=C.UTF-8")
+	}
+	return env
+}
+
 func (l Local) lookPath(env Environment, dir string, program string) (string, error) {
 	abs := func(path string) string {
 		if filepath.IsAbs(path) {

internal/biome/biome_test.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -157,6 +159,84 @@ func TestLocal(t *testing.T) {
 	}
 }
 
+func TestStandardEnv(t *testing.T) {
+	stdenv := appendStandardEnv(nil, runtime.GOOS)
+
+	t.Run("TZ", func(t *testing.T) {
+		found := false
+		for _, e := range stdenv {
+			const prefix = "TZ="
+			if !strings.HasPrefix(e, prefix) {
+				continue
+			}
+			found = true
+			if got, want := e[len(prefix):], "UTC0"; got != want {
+				t.Errorf("TZ = %q; want %q", got, want)
+			}
+		}
+		if !found {
+			t.Error("TZ not set")
+		}
+	})
+
+	t.Run("LANG", func(t *testing.T) {
+		found := false
+		for _, e := range stdenv {
+			const prefix = "LANG="
+			if !strings.HasPrefix(e, prefix) {
+				continue
+			}
+			found = true
+			if got, want1, want2 := e[len(prefix):], "C.UTF-8", "C"; got != want1 && got != want2 {
+				t.Errorf("LANG = %q; want %q or %q", got, want1, want2)
+			}
+		}
+		if !found {
+			t.Error("LANG not set")
+		}
+	})
+
+	t.Run("Charmap", func(t *testing.T) {
+		// Run locale tool to get character encoding.
+		// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/locale.html
+		c := exec.Command("locale", "-k", "charmap")
+		c.Env = stdenv
+		stdout := new(strings.Builder)
+		stderr := new(strings.Builder)
+		c.Stdout = stdout
+		c.Stderr = stderr
+		err := c.Run()
+		if stderr.Len() > 0 {
+			t.Logf("stderr:\n%s", stderr)
+		}
+		if err != nil {
+			t.Error("locale:", err)
+		}
+		got := parseLocaleOutput(stdout.String())["charmap"]
+		const want = "UTF-8"
+		if got != want {
+			t.Errorf("charmap = %q; want %q", got, want)
+		}
+	})
+}
+
+func parseLocaleOutput(out string) map[string]string {
+	m := make(map[string]string)
+	for _, line := range strings.Split(out, "\n") {
+		eq := strings.Index(line, "=")
+		if eq == -1 {
+			continue
+		}
+		k, v := line[:eq], line[eq+1:]
+		if strings.HasPrefix(v, `"`) {
+			v = strings.TrimPrefix(v, `"`)
+			v = strings.TrimSuffix(v, `"`)
+		}
+		m[k] = v
+	}
+	return m
+}
+
 func TestExecPrefix(t *testing.T) {
 	tests := []struct {
 		name        string

internal/biome/docker.go

@@ -240,8 +240,8 @@ func (c *Container) Run(ctx context.Context, invoke *Invocation) error {
 	opts.Env = []string{
 		// TODO(light): Set LOGNAME and USER.
 		"HOME=" + c.dirs.Home,
-		"TZ=UTC",
 	}
+	opts.Env = appendStandardEnv(opts.Env, c.Describe().OS)
 	opts.Env = invoke.Env.appendTo(opts.Env, c.path, ':')
 	if slashpath.IsAbs(invoke.Dir) {
 		opts.WorkingDir = invoke.Dir