Skip to content

Commit

Permalink
chore: merge 24.05.2 content (#2984)
Browse files Browse the repository at this point in the history
* fix: fixing non-exported flexgroup instances error

* fix: Harvest should log errors when grafana import fails (#2962)

Thanks to Chris Gautcher (@gautcher) for raising.

* ci: bump go (#2965)

* chore: merge 24.05.2 content

* fix: Rest templates should not have hyphon (#2943)

* fix: Rest templates should not have hyphon

* fix: address review comments

* chore: fix ci

---------

Co-authored-by: hardikl <[email protected]>
Co-authored-by: Chris Grindstaff <[email protected]>
  • Loading branch information
3 people authored Jun 13, 2024
1 parent 1e2eb72 commit 2054743
Show file tree
Hide file tree
Showing 10 changed files with 42 additions and 47 deletions.
2 changes: 1 addition & 1 deletion .harvest.env
Original file line number Diff line number Diff line change
@@ -1 +1 @@
GO_VERSION=1.22.3
GO_VERSION=1.22.4
18 changes: 3 additions & 15 deletions cmd/collectors/restperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util

fgAggrMap := make(map[string]*set.Set)
flexgroupAggrsMap := make(map[string]*set.Set)
nonExportedInstanceMap := make(map[string]bool)

// volume_aggr_labels metric is deprecated now and will be removed later.
metricName := "labels"
Expand All @@ -69,11 +68,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
cache.UUID += ".Volume"

// create flexgroup instance cache
for iKey, i := range data.GetInstances() {
if !i.IsExportable() {
nonExportedInstanceMap[iKey] = true
continue
}
for _, i := range data.GetInstances() {
if match := re.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
// instance key is svm.flexgroup-volume
key := i.GetLabel("svm") + "." + match[1]
Expand Down Expand Up @@ -102,10 +97,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
fgAggrMap[key].Add(i.GetLabel("aggr"))
flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
i.SetLabel(style, "flexgroup_constituent")
if !v.includeConstituents {
i.SetExportable(false)
nonExportedInstanceMap[iKey] = true
}
i.SetExportable(v.includeConstituents)
} else {
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + i.GetLabel("volume")
Expand All @@ -127,11 +119,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
// cache.Reset()

// create summary
for iKey, i := range data.GetInstances() {
if nonExportedInstanceMap[iKey] {
continue
}

for _, i := range data.GetInstances() {
match := re.FindStringSubmatch(i.GetLabel("volume"))
if len(match) != 3 {
continue
Expand Down
19 changes: 3 additions & 16 deletions cmd/collectors/zapiperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util

fgAggrMap := make(map[string]*set.Set)
flexgroupAggrsMap := make(map[string]*set.Set)
nonExportedInstanceMap := make(map[string]bool)

// volume_aggr_labels metric is deprecated now and will be removed later.
metricName := "labels"
Expand All @@ -77,11 +76,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
cache.UUID += ".Volume"

// create flexgroup instance cache
for iKey, i := range data.GetInstances() {
if !i.IsExportable() {
nonExportedInstanceMap[iKey] = true
continue
}
for _, i := range data.GetInstances() {
if match := re.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
// instance key is svm.flexgroup-volume
key := i.GetLabel("svm") + "." + match[1]
Expand Down Expand Up @@ -110,10 +105,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
fgAggrMap[key].Add(i.GetLabel("aggr"))
flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
i.SetLabel(style, "flexgroup_constituent")
if !v.includeConstituents {
i.SetExportable(false)
nonExportedInstanceMap[iKey] = true
}
i.SetExportable(v.includeConstituents)
} else {
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + i.GetLabel("volume")
Expand All @@ -128,19 +120,14 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
v.Logger.Error().Err(err).Str("metric", metricName).Msg("Unable to set value on metric")
}
}

}

v.Logger.Debug().Msgf("extracted %d flexgroup volumes", len(cache.GetInstances()))

// cache.Reset()

// create summary
for iKey, i := range data.GetInstances() {
if nonExportedInstanceMap[iKey] {
continue
}

for _, i := range data.GetInstances() {
match := re.FindStringSubmatch(i.GetLabel("volume"))
if len(match) != 3 {
continue
Expand Down
15 changes: 9 additions & 6 deletions cmd/tools/grafana/grafana.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ func doImport(_ *cobra.Command, _ []string) {
opts.command = "import"
_, err := conf.LoadHarvestConfig(opts.config)
if err != nil {
return
printErrorAndExit(err)
}

adjustOptions()
Expand All @@ -376,6 +376,11 @@ func doImport(_ *cobra.Command, _ []string) {
importDashboards(opts)
}

func printErrorAndExit(err error) {
fmt.Println(err)
os.Exit(1)
}

func validateImport() {
// default case
if opts.dir == "" && opts.serverfolder.name == "" {
Expand All @@ -401,7 +406,7 @@ func initImportVars() {
// default behaviour
if opts.dir == "grafana/dashboards" && opts.serverfolder.name == "" {
m[filepath.Join(opts.dir, "cmode")] = &Folder{name: "Harvest-" + harvestRelease + "-cDOT"}
m[filepath.Join(opts.dir, "cmode", "details")] = &Folder{name: "Harvest-" + harvestRelease + "-cDOT Details"}
m[filepath.Join(opts.dir, "cmode-details")] = &Folder{name: "Harvest-" + harvestRelease + "-cDOT Details"}
m[filepath.Join(opts.dir, "7mode")] = &Folder{name: "Harvest-" + harvestRelease + "-7mode"}
m[filepath.Join(opts.dir, "storagegrid")] = &Folder{name: "Harvest-" + harvestRelease + "-StorageGrid"}
} else if opts.dir != "" && opts.serverfolder.name != "" {
Expand All @@ -412,8 +417,7 @@ func initImportVars() {
if opts.customizeDir == "" {
err := checkAndCreateServerFolder(v)
if err != nil {
fmt.Print(err)
os.Exit(1)
printErrorAndExit(err)
}
}
opts.dirGrafanaFolderMap[k] = v
Expand Down Expand Up @@ -469,8 +473,7 @@ func importFiles(dir string, folder *Folder) {
return
}
if files, err = os.ReadDir(dir); err != nil {
// TODO check for not exist
return
printErrorAndExit(err)
}

for _, file := range files {
Expand Down
16 changes: 16 additions & 0 deletions cmd/tools/template/template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,22 @@ func TestOverrideMetricsExist(t *testing.T) {
}, allTemplatesButEms...)
}

func TestNoHyphenInMetrics(t *testing.T) {
visitTemplates(t, func(path string, model Model) {
isRest := strings.Contains(path, "rest")

if !isRest {
return
}

for _, m := range model.metrics {
if strings.Contains(m.left, "-") || strings.Contains(m.right, "-") {
t.Errorf("Metric fields 'left' and 'right' should not contain hyphens. Found in path=%s, left=%s, right=%s", shortPath(path), m.left, m.right)
}
}
}, allTemplatesButEms...)
}

type sorted struct {
got string
want string
Expand Down
2 changes: 1 addition & 1 deletion conf/rest/9.12.0/volume.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ counters:
- files => inode_files_total
- files_used => inode_files_used
- filesystem_size => filesystem_size
- logical-available => space_logical_available
- logical_available => space_logical_available
- logical_used => space_logical_used
- logical_used_by_afs => space_logical_used_by_afs
- logical_used_by_snapshots => space_logical_used_by_snapshots
Expand Down
5 changes: 5 additions & 0 deletions integration/test/dashboard_json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ var zapiCounterMap = map[string]struct{}{
"volume_arw_status": {},
"volume_num_compress_fail": {},
"volume_num_compress_attempts": {},
// sar is experiencing high api time for ZapiPerf. The u2 cluster does not have fabricpool added for the collection of these counters. Remove the following once sar is capable of running ZapiPerf.
"volume_performance_tier_footprint": {},
"volume_capacity_tier_footprint": {},
"volume_capacity_tier_footprint_percent": {},
"volume_performance_tier_footprint_percent": {},
}

// restCounterMap are additional counters, above and beyond the ones from counterMap, which should be excluded from Rest
Expand Down
2 changes: 1 addition & 1 deletion integration/test/dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func TestGrafanaAndPrometheusAreConfigured(t *testing.T) {
cDotFolder = "Harvest-" + version.VERSION + "-cDOT"
sevenModeFolder = "Harvest-" + version.VERSION + "-7mode"
log.Info().Str("cMode", cDotFolder).Str("7mode", sevenModeFolder).Msg("Folder name details")
status, out := new(grafana.Mgr).Import("") // send empty so that it will import all dashboards
status, out := new(grafana.Mgr).Import()
if !status {
t.Errorf("Grafana import operation failed out=%s", out)
}
Expand Down
8 changes: 2 additions & 6 deletions integration/test/grafana/grafana_mgr.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
type Mgr struct {
}

func (g *Mgr) Import(jsonDir string) (bool, string) {
func (g *Mgr) Import() (bool, string) {
var (
importOutput string
status bool
Expand All @@ -31,15 +31,11 @@ func (g *Mgr) Import(jsonDir string) (bool, string) {
if err != nil {
panic(err)
}
directoryOption := ""
if jsonDir != "" {
directoryOption = "--directory"
}
grafanaURL := utils.GetGrafanaURL()
if docker.IsDockerBasedPoller() {
grafanaURL = "grafana:3000"
}
importCmds := []string{"grafana", "import", "--overwrite", "--addr", grafanaURL, directoryOption, jsonDir}
importCmds := []string{"grafana", "import", "--overwrite", "--addr", grafanaURL}
if docker.IsDockerBasedPoller() {
params := []string{"exec", containerIDs[0].ID, "bin/harvest"}
params = append(params, importCmds...)
Expand Down
2 changes: 1 addition & 1 deletion integration/test/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func IsURLReachable(url string) bool {
return false
}
defer response.Body.Close()
return response.StatusCode == 200
return response.StatusCode == http.StatusOK
}

func AddPrometheusToGrafana() {
Expand Down

0 comments on commit 2054743

Please sign in to comment.