Introduce additional checks for dhall (deps, dirtyWhen)

buildkite/Makefile

@@ -14,4 +14,17 @@ lint:
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii lint --inplace {} || exit 255'
 
 format:
-	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255'
+	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255'
+
+
+check_deps:
+	 $(eval TMP := $(shell mktemp -d))
+	scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)"
+	python3 scripts/dhall/checker.py --root "$(TMP)" deps
+
+check_dirty:
+	$(eval TMP := $(shell mktemp -d))
+	scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)"
+	python3 scripts/dhall/checker.py --root "$(TMP)" dirty-when  --repo "$(PWD)/../"
+
+all: check_syntax lint format check_deps check_dirty

buildkite/scripts/dhall/checker.py

@@ -0,0 +1,206 @@
+"""
+    Runs dhall checks like:
+
+    - validate if all dependencies in jobs are covered
+
+        python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs deps
+
+    - all dirtyWhen entries relates to existing files
+
+        python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs dirty-when
+
+    - print commands for given job
+
+        python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs print-cmd --job SingleNodeTest
+"""
+
+
+import argparse
+import subprocess
+import os
+from glob import glob
+import tempfile
+from pathlib import Path
+import yaml
+
+
+class CmdColors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+class PipelineInfoBuilder:
+
+    def __init__(self, temp, file):
+        with open(f"{temp}/{file}") as stream:
+            try:
+                self.pipeline = yaml.safe_load(stream)
+                self.file = file
+            except yaml.YAMLError as exc:
+                print(f"cannot parse correctly {temp}/{file}, due to {exc}")
+                exit(1)
+
+    def get_steps(self):
+        steps = []
+        for step in self.pipeline["pipeline"]["steps"]:
+            key = step["key"]
+            deps = []
+            if "depends_on" in step:
+                for dependsOn in step["depends_on"]:
+                    deps.append(dependsOn["step"])
+            commands = step["commands"]
+            steps.append(Step(key, deps, commands))
+        return steps
+
+    def get_dirty(self):
+        dirty = []
+        for dirtyWhen in self.pipeline["spec"]["dirtyWhen"]:
+            path = dirtyWhen["dir"][0] if "dir" in dirtyWhen else ""
+            exts = dirtyWhen["exts"][0] if "exts" in dirtyWhen else ""
+            strictEnd = bool(dirtyWhen["strictEnd"]) if (
+                "strictEnd" in dirtyWhen) else False
+            strictStart = bool(dirtyWhen["strictStart"]) if (
+                "strictStart" in dirtyWhen) else False
+            dirty.append(DirtyWhen(path=path, strictStart=strictStart,
+                         strictEnd=strictEnd, extension=exts))
+        return dirty
+
+    def build(self):
+        steps = self.get_steps()
+        dirty = self.get_dirty()
+        return PipelineInfo(self.file, self.pipeline, steps, dirty)
+
+
+class DirtyWhen:
+
+    def __init__(self, path, extension, strictStart, strictEnd):
+        self.path = path
+        self.extension = extension
+        self.strictStart = strictStart
+        self.strictEnd = strictEnd
+
+    def calculate_path(self,repo):
+        if not self.path:
+            return glob(os.path.join(repo,f'**/*{self.extension}'))
+        if not self.extension:
+            if self.strictEnd and self.strictStart:
+                return glob(os.path.join(repo, f'{self.path}'))
+            if not self.strictEnd and self.strictStart:
+                return glob(os.path.join(repo, f'{self.path}*'))
+            if not self.strictStart and self.strictEnd:
+                return glob(os.path.join(repo, f'**/{self.path}'), recursive= True)
+            if not self.strictStart and not self.strictEnd:
+                return glob(os.path.join(repo, f'*{self.path}*'))
+        return glob(os.path.join(repo, f'{self.path}.{self.extension}'))
+
+    def __str__(self):
+        return f"path: '{self.path}', exts: '{self.extension}', startStrict:{self.strictStart}, startEnd:{self.strictEnd}"
+
+
+class Step:
+
+    def __init__(self, key, deps, commands):
+        self.key = key
+        self.deps = deps
+        self.commands = commands
+
+
+class PipelineInfo:
+
+    def __init__(self, file, pipeline, steps, dirty):
+        self.file = file
+        self.pipeline = pipeline
+        self.steps = steps
+        self.dirty = dirty
+
+    def keys(self):
+        return [step.key for step in self.steps]
+
+
+parser = argparse.ArgumentParser(description='Executes mina benchmarks')
+parser.add_argument("--root", required=True,
+                    help="root folder where all dhall files resides")
+
+subparsers = parser.add_subparsers(dest="cmd")
+dirty_when = subparsers.add_parser('dirty-when')
+dirty_when.add_argument("--repo", required=True,
+                    help="root folder for mina repo")
+
+subparsers.add_parser('deps')
+
+
+run = subparsers.add_parser('print-cmd')
+run.add_argument("--job", required=True, help="job to run")
+run.add_argument("--step", required=False, help="job to run")
+
+
+args = parser.parse_args()
+
+pipelinesInfo = [PipelineInfoBuilder(args.root, file).build()
+                 for file in os.listdir(path=args.root)]
+
+if args.cmd == "deps":
+
+    keys = []
+    for pipeline in pipelinesInfo:
+        keys.extend(pipeline.keys())
+
+    failedSteps = []
+
+    for pipeline in pipelinesInfo:
+        for step in pipeline.steps:
+            for dep in step.deps:
+                if not dep in keys:
+                    failedSteps.append((pipeline, step, dep))
+
+    if any(failedSteps):
+        print("Fatal: Missing dependency resolution found:")
+        for (pipeline, step, dep) in failedSteps:
+            file = str.replace(pipeline.file, ".yml", ".dhall")
+            print(
+                f"\t{CmdColors.FAIL}[FATAL] Unresolved dependency for step '{step.key}' in '{file}' depends on non existing job '{dep}'{CmdColors.ENDC}")
+        exit(1)
+    else:
+        print('Pipelines definitions correct')
+
+if args.cmd == "print-cmd":
+    pipeline = next(filter(lambda x: args.job in x.file, pipelinesInfo))
+
+    def get_steps():
+        if args.step:
+            return [next(filter(lambda x: args.step in x.key, pipeline.steps))]
+        else:
+            return pipeline.steps
+
+    steps = get_steps()
+
+    for step in steps:
+        for command in step.commands:
+            if not command.startswith("echo"):
+                print(command)
+
+if args.cmd == "dirty-when":
+
+    failedSteps = []
+
+    for pipeline in pipelinesInfo:
+        for dirty in pipeline.dirty:
+            if not bool(dirty.calculate_path(args.repo)):
+                failedSteps.append((pipeline, dirty))
+
+    if any(failedSteps):
+        print("Fatal: Non existing dirtyWhen path detected:")
+        for (pipeline, dirty) in failedSteps:
+            file = str.replace(pipeline.file, ".yml", ".dhall")
+            print(
+                f"\t{CmdColors.FAIL}[FATAL] Unresolved dirtyWhen path  in '{file}' ('{str(dirty)}'){CmdColors.ENDC}")
+        exit(1)
+    else:
+        print('Pipelines definitions correct')

buildkite/scripts/dhall/dump_dhall_to_pipelines.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+ROOT=$1
+OUTPUT=$2
+
+mkdir -p "$OUTPUT"
+
+shopt -s globstar nullglob
+
+echo "Dumping pipelines from '$ROOT' to '$OUTPUT'"
+
+COUNTER=0
+
+for file in "$ROOT"/**/*.dhall
+do
+    filename=$(basename "$file")
+    filename="${filename%.*}"
+
+    dhall-to-yaml --quoted --file "$file" > "$OUTPUT"/"$filename".yml
+
+    COUNTER=$((COUNTER+1))
+done
+
+echo "Done. $COUNTER jobs exported"

buildkite/src/Jobs/Lint/Dhall.dhall

@@ -14,6 +14,15 @@ let Docker = ../../Command/Docker/Type.dhall
 
 let Size = ../../Command/Size.dhall
 
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
+let dump_pipelines_cmd =
+      Cmd.runInDocker
+        Cmd.Docker::{
+        , image = (../../Constants/ContainerImages.dhall).toolchainBase
+        }
+        "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines"
+
 in  Pipeline.build
       Pipeline.Config::{
       , spec = JobSpec::{
@@ -54,5 +63,29 @@ in  Pipeline.build
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }
             }
+        , Command.build
+            Command.Config::{
+            , commands =
+                  [ dump_pipelines_cmd ]
+                # RunInToolchain.runInToolchainBullseye
+                    ([] : List Text)
+                    "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines deps"
+            , label = "Dhall: deps"
+            , key = "check-dhall-deps"
+            , target = Size.Multi
+            , docker = None Docker.Type
+            }
+        , Command.build
+            Command.Config::{
+            , commands =
+                  [ dump_pipelines_cmd ]
+                # RunInToolchain.runInToolchainBullseye
+                    ([] : List Text)
+                    "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines dirty-when  --repo ."
+            , label = "Dhall: dirtyWhen"
+            , key = "check-dhall-dirty"
+            , target = Size.Multi
+            , docker = None Docker.Type
+            }
         ]
       }

buildkite/src/Jobs/Lint/ValidationService.dhall

@@ -75,10 +75,7 @@ in  Pipeline.build
                   (S.contains "buildkite/src/Jobs/Lint/ValidationService")
 
           in  JobSpec::{
-              , dirtyWhen =
-                [ dirtyDhallDir
-                , S.strictlyStart (S.contains ValidationService.rootPath)
-                ]
+              , dirtyWhen = [ dirtyDhallDir ]
               , path = "Lint"
               , name = "ValidationService"
               , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]

buildkite/src/Jobs/Lint/Xrefcheck.dhall

@@ -23,7 +23,7 @@ in  Pipeline.build
       , spec = JobSpec::{
         , dirtyWhen =
           [ SelectFiles.strictly SelectFiles::{ exts = Some [ "md" ] }
-          , SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml")
+          , SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yaml")
           ]
         , path = "Lint"
         , name = "Xrefcheck"

buildkite/src/Jobs/Release/TestnetAlerts.dhall

@@ -40,8 +40,6 @@ in  Pipeline.build
             , label = "Deploy Testnet alert rules"
             , key = "deploy-testnet-alerts"
             , target = Size.Medium
-            , depends_on =
-              [ { name = "TestnetAlerts", key = "lint-testnet-alerts" } ]
             , docker = None Docker.Type
             , if = Some "build.env('DEPLOY_ALERTS') == 'true'"
             }

buildkite/src/Jobs/Test/TerraformNetworkTest.dhall

@@ -35,8 +35,8 @@ in  Pipeline.build
       Pipeline.Config::{
       , spec =
           let unitDirtyWhen =
-                [ S.strictlyStart (S.contains "src/automation/terraform")
-                , S.strictlyStart (S.contains "src/helm")
+                [ S.strictlyStart (S.contains "automation/terraform")
+                , S.strictlyStart (S.contains "helm")
                 , S.strictlyStart
                     (S.contains "buildkite/src/Jobs/Test/TerraformNetworkTest")
                 , S.strictlyStart

buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall

@@ -38,20 +38,15 @@ in  Pipeline.build
           , S.strictlyStart (S.contains "dockerfiles")
           , S.strictlyStart
               (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest")
-          , S.strictlyStart
-              (S.contains "buildkite/src/Jobs/Command/TestExecutive")
+          , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive")
           , S.strictlyStart
               (S.contains "automation/terraform/modules/o1-integration")
           , S.strictlyStart
               (S.contains "automation/terraform/modules/kubernetes/testnet")
           , S.strictlyStart
-              ( S.contains
-                  "automation/buildkite/script/run-test-executive-cloud"
-              )
+              (S.contains "buildkite/scripts/run-test-executive-cloud")
           , S.strictlyStart
-              ( S.contains
-                  "automation/buildkite/script/run-test-executive-local"
-              )
+              (S.contains "buildkite/scripts/run-test-executive-local")
           ]
         , path = "Test"
         , name = "TestnetIntegrationTests"

buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall

@@ -38,8 +38,7 @@ in  Pipeline.build
           , S.strictlyStart (S.contains "dockerfiles")
           , S.strictlyStart
               (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest")
-          , S.strictlyStart
-              (S.contains "buildkite/src/Jobs/Command/TestExecutive")
+          , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive")
           , S.strictlyStart
               (S.contains "automation/terraform/modules/o1-integration")
           , S.strictlyStart