diff --git a/buildkite/Makefile b/buildkite/Makefile index 33de08c0af8..5464c602687 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -14,4 +14,16 @@ lint: find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii lint --inplace {} || exit 255' format: - find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' \ No newline at end of file + find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' + +check_deps: + $(eval TMP := $(shell mktemp -d)) + scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" + python3 scripts/dhall/checker.py --root "$(TMP)" deps + +check_dirty: + $(eval TMP := $(shell mktemp -d)) + scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" + python3 scripts/dhall/checker.py --root "$(TMP)" dirty-when --repo "$(PWD)/../" + +all: check_syntax lint format check_deps check_dirty \ No newline at end of file diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py new file mode 100755 index 00000000000..df64bc45092 --- /dev/null +++ b/buildkite/scripts/dhall/checker.py @@ -0,0 +1,206 @@ +""" + Runs dhall checks like: + + - validate if all dependencies in jobs are covered + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs deps + + - all dirtyWhen entries relates to existing files + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs dirty-when + + - print commands for given job + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs print-cmd --job SingleNodeTest +""" + + +import argparse +import subprocess +import os +from glob import glob +import tempfile +from pathlib import Path +import yaml + + +class CmdColors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +class PipelineInfoBuilder: + + def __init__(self, temp, file): + with open(f"{temp}/{file}") as stream: + try: + self.pipeline = yaml.safe_load(stream) + self.file = file + except yaml.YAMLError as exc: + print(f"cannot parse correctly {temp}/{file}, due to {exc}") + exit(1) + + def get_steps(self): + steps = [] + for step in self.pipeline["pipeline"]["steps"]: + key = step["key"] + deps = [] + if "depends_on" in step: + for dependsOn in step["depends_on"]: + deps.append(dependsOn["step"]) + commands = step["commands"] + steps.append(Step(key, deps, commands)) + return steps + + def get_dirty(self): + dirty = [] + for dirtyWhen in self.pipeline["spec"]["dirtyWhen"]: + path = dirtyWhen["dir"][0] if "dir" in dirtyWhen else "" + exts = dirtyWhen["exts"][0] if "exts" in dirtyWhen else "" + strictEnd = bool(dirtyWhen["strictEnd"]) if ( + "strictEnd" in dirtyWhen) else False + strictStart = bool(dirtyWhen["strictStart"]) if ( + "strictStart" in dirtyWhen) else False + dirty.append(DirtyWhen(path=path, strictStart=strictStart, + strictEnd=strictEnd, extension=exts)) + return dirty + + def build(self): + steps = self.get_steps() + dirty = self.get_dirty() + return PipelineInfo(self.file, self.pipeline, steps, dirty) + + +class DirtyWhen: + + def __init__(self, path, extension, strictStart, strictEnd): + self.path = path + self.extension = extension + self.strictStart = strictStart + self.strictEnd = strictEnd + + def calculate_path(self,repo): + if not self.path: + return glob(os.path.join(repo,f'**/*{self.extension}')) + if not self.extension: + if self.strictEnd and self.strictStart: + return glob(os.path.join(repo, f'{self.path}')) + if not self.strictEnd and self.strictStart: + return glob(os.path.join(repo, f'{self.path}*')) + if not self.strictStart and self.strictEnd: + return glob(os.path.join(repo, f'**/{self.path}'), recursive= True) + if not self.strictStart and not self.strictEnd: + return glob(os.path.join(repo, f'*{self.path}*')) + return glob(os.path.join(repo, f'{self.path}.{self.extension}')) + + def __str__(self): + return f"path: '{self.path}', exts: '{self.extension}', startStrict:{self.strictStart}, startEnd:{self.strictEnd}" + + +class Step: + + def __init__(self, key, deps, commands): + self.key = key + self.deps = deps + self.commands = commands + + +class PipelineInfo: + + def __init__(self, file, pipeline, steps, dirty): + self.file = file + self.pipeline = pipeline + self.steps = steps + self.dirty = dirty + + def keys(self): + return [step.key for step in self.steps] + + +parser = argparse.ArgumentParser(description='Executes mina benchmarks') +parser.add_argument("--root", required=True, + help="root folder where all dhall files resides") + +subparsers = parser.add_subparsers(dest="cmd") +dirty_when = subparsers.add_parser('dirty-when') +dirty_when.add_argument("--repo", required=True, + help="root folder for mina repo") + +subparsers.add_parser('deps') + + +run = subparsers.add_parser('print-cmd') +run.add_argument("--job", required=True, help="job to run") +run.add_argument("--step", required=False, help="job to run") + + +args = parser.parse_args() + +pipelinesInfo = [PipelineInfoBuilder(args.root, file).build() + for file in os.listdir(path=args.root)] + +if args.cmd == "deps": + + keys = [] + for pipeline in pipelinesInfo: + keys.extend(pipeline.keys()) + + failedSteps = [] + + for pipeline in pipelinesInfo: + for step in pipeline.steps: + for dep in step.deps: + if not dep in keys: + failedSteps.append((pipeline, step, dep)) + + if any(failedSteps): + print("Fatal: Missing dependency resolution found:") + for (pipeline, step, dep) in failedSteps: + file = str.replace(pipeline.file, ".yml", ".dhall") + print( + f"\t{CmdColors.FAIL}[FATAL] Unresolved dependency for step '{step.key}' in '{file}' depends on non existing job '{dep}'{CmdColors.ENDC}") + exit(1) + else: + print('Pipelines definitions correct') + +if args.cmd == "print-cmd": + pipeline = next(filter(lambda x: args.job in x.file, pipelinesInfo)) + + def get_steps(): + if args.step: + return [next(filter(lambda x: args.step in x.key, pipeline.steps))] + else: + return pipeline.steps + + steps = get_steps() + + for step in steps: + for command in step.commands: + if not command.startswith("echo"): + print(command) + +if args.cmd == "dirty-when": + + failedSteps = [] + + for pipeline in pipelinesInfo: + for dirty in pipeline.dirty: + if not bool(dirty.calculate_path(args.repo)): + failedSteps.append((pipeline, dirty)) + + if any(failedSteps): + print("Fatal: Non existing dirtyWhen path detected:") + for (pipeline, dirty) in failedSteps: + file = str.replace(pipeline.file, ".yml", ".dhall") + print( + f"\t{CmdColors.FAIL}[FATAL] Unresolved dirtyWhen path in '{file}' ('{str(dirty)}'){CmdColors.ENDC}") + exit(1) + else: + print('Pipelines definitions correct') diff --git a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh new file mode 100755 index 00000000000..84193329b76 --- /dev/null +++ b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +ROOT=$1 +OUTPUT=$2 + +mkdir -p "$OUTPUT" + +shopt -s globstar nullglob + +echo "Dumping pipelines from '$ROOT' to '$OUTPUT'" + +COUNTER=0 + +for file in "$ROOT"/**/*.dhall +do + filename=$(basename "$file") + filename="${filename%.*}" + + dhall-to-yaml --quoted --file "$file" > "$OUTPUT"/"$filename".yml + + COUNTER=$((COUNTER+1)) +done + +echo "Done. $COUNTER jobs exported" diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 481da3c8817..687e2f1c9d9 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -14,6 +14,15 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall + +let dump_pipelines_cmd = + Cmd.runInDocker + Cmd.Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } + "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -58,5 +67,29 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } + , Command.build + Command.Config::{ + , commands = + [ dump_pipelines_cmd ] + # RunInToolchain.runInToolchainBullseye + ([] : List Text) + "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines deps" + , label = "Dhall: deps" + , key = "check-dhall-deps" + , target = Size.Multi + , docker = None Docker.Type + } + , Command.build + Command.Config::{ + , commands = + [ dump_pipelines_cmd ] + # RunInToolchain.runInToolchainBullseye + ([] : List Text) + "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines dirty-when --repo ." + , label = "Dhall: dirtyWhen" + , key = "check-dhall-dirty" + , target = Size.Multi + , docker = None Docker.Type + } ] } diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index fcff4b30627..350de107453 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -75,10 +75,7 @@ in Pipeline.build (S.contains "buildkite/src/Jobs/Lint/ValidationService") in JobSpec::{ - , dirtyWhen = - [ dirtyDhallDir - , S.strictlyStart (S.contains ValidationService.rootPath) - ] + , dirtyWhen = [ dirtyDhallDir ] , path = "Lint" , name = "ValidationService" , tags = diff --git a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall index 336f4612cb6..3108316b34b 100644 --- a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall +++ b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall @@ -35,8 +35,8 @@ in Pipeline.build Pipeline.Config::{ , spec = let unitDirtyWhen = - [ S.strictlyStart (S.contains "src/automation/terraform") - , S.strictlyStart (S.contains "src/helm") + [ S.strictlyStart (S.contains "automation/terraform") + , S.strictlyStart (S.contains "helm") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TerraformNetworkTest") , S.strictlyStart diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 933a21bae6e..a519b2f4016 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -38,20 +38,15 @@ in Pipeline.build , S.strictlyStart (S.contains "dockerfiles") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") - , S.strictlyStart - (S.contains "buildkite/src/Jobs/Command/TestExecutive") + , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") , S.strictlyStart - ( S.contains - "automation/buildkite/script/run-test-executive-cloud" - ) + (S.contains "buildkite/scripts/run-test-executive-cloud") , S.strictlyStart - ( S.contains - "automation/buildkite/script/run-test-executive-local" - ) + (S.contains "buildkite/scripts/run-test-executive-local") ] , path = "Test" , name = "TestnetIntegrationTests" diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index c02eb9c5262..4047d2212fd 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -38,8 +38,7 @@ in Pipeline.build , S.strictlyStart (S.contains "dockerfiles") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") - , S.strictlyStart - (S.contains "buildkite/src/Jobs/Command/TestExecutive") + , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart