[Master] Patch missing type shapes

buildkite/scripts/dump-mina-type-shapes.sh

@@ -20,3 +20,5 @@ export TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1}-type_shape.txt
 
 echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1}"
 mina internal dump-type-shapes > ${TYPE_SHAPE_FILE}
+
+source buildkite/scripts/gsutil-upload.sh ${TYPE_SHAPE_FILE} gs://mina-type-shapes

buildkite/scripts/gsutil-upload.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+KEY_FILE=/var/secrets/google/key.json
+
+if [ ! -f $KEY_FILE ]; then
+    echo "Cannot use gsutil for upload as key file cannot be foud in $KEY_FILE"
+fi
+
+gcloud auth activate-service-account --key-file=$KEY_FILE
+
+gsutil cp $1 $2

buildkite/scripts/version-linter-patch-missing-type-shapes.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -eox pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <release-branch>"
+    exit 1
+fi
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/handle-fork.sh
+source buildkite/scripts/export-git-env-vars.sh
+
+release_branch=${REMOTE}/$1
+
+RELEASE_BRANCH_COMMIT=$(git log -n 1 --format="%h" --abbrev=7 $release_branch)
+
+function revert_checkout() {
+    git checkout $BUILDKITE_COMMIT
+    git submodule sync
+    git submodule update --init --recursive
+}
+
+function checkout_and_dump() {
+    local __commit=$1
+    git checkout $__commit
+    git submodule sync
+    git submodule update --init --recursive
+    eval $(opam config env)
+    TYPE_SHAPE_FILE=${__commit:0:7}-type_shape.txt
+    dune exec src/app/cli/src/mina.exe internal dump-type-shapes > /tmp/${TYPE_SHAPE_FILE}
+    revert_checkout
+    source buildkite/scripts/gsutil-upload.sh /tmp/${TYPE_SHAPE_FILE} gs://mina-type-shapes
+}
+
+if ! $(gsutil ls gs://mina-type-shapes/$BUILDKITE_COMMIT 2>/dev/null); then
+    checkout_and_dump $BUILDKITE_COMMIT
+fi
+
+if ! $(gsutil ls gs://mina-type-shapes/$RELEASE_BRANCH_COMMIT 2>/dev/null); then
+    checkout_and_dump $RELEASE_BRANCH_COMMIT
+fi
+
+if [[ -n "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}" ]]; then 
+    BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT=$(git log -n 1 --format="%h" --abbrev=7 ${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH} )
+    if ! $(gsutil ls gs://mina-type-shapes/$BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT 2>/dev/null); then
+        checkout_and_dump $BUILDKITE_PULL_REQUEST_BASE_BRANCH_COMMIT
+    fi
+fi

buildkite/src/Jobs/Test/VersionLint.dhall

@@ -1,5 +1,3 @@
-let Cmd = ../../Lib/Cmds.dhall
-
 let S = ../../Lib/SelectFiles.dhall
 
 let B = ../../External/Buildkite.dhall
@@ -14,12 +12,12 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
 
-let RunInToolchain = ../../Command/RunInToolchain.dhall
-
 let Docker = ../../Command/Docker/Type.dhall
 
 let Size = ../../Command/Size.dhall
 
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
 let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ]
 
 let buildTestCmd
@@ -34,18 +32,18 @@ let buildTestCmd
                   RunInToolchain.runInToolchain
                     ([] : List Text)
                     "buildkite/scripts/dump-mina-type-shapes.sh"
-                # [ Cmd.run
-                      "gsutil cp \$(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt \$MINA_TYPE_SHAPE gs://mina-type-shapes"
-                  ]
+                # RunInToolchain.runInToolchain
+                    ([] : List Text)
+                    "buildkite/scripts/version-linter-patch-missing-type-shapes.sh ${release_branch}"
                 # RunInToolchain.runInToolchain
                     ([] : List Text)
                     "buildkite/scripts/version-linter.sh ${release_branch}"
             , label = "Versioned type linter for ${release_branch}"
             , key = "version-linter-${release_branch}"
-            , soft_fail = Some soft_fail
             , target = cmd_target
             , docker = None Docker.Type
             , depends_on = dependsOn
+            , soft_fail = Some soft_fail
             , artifact_paths = [ S.contains "core_dumps/*" ]
             }
 

buildkite/src/Lib/Cmds.dhall

@@ -13,8 +13,16 @@ let module =
           \(environment : List Text)
       ->  let Docker =
                 { Type =
-                    { image : Text, extraEnv : List Text, privileged : Bool }
-                , default = { extraEnv = [] : List Text, privileged = False }
+                    { image : Text
+                    , extraEnv : List Text
+                    , privileged : Bool
+                    , useBash : Bool
+                    }
+                , default =
+                    { extraEnv = [] : List Text
+                    , privileged = False
+                    , useBash = True
+                    }
                 }
 
           let Cmd = { line : Text, readable : Optional Text }
@@ -58,12 +66,16 @@ let module =
                         : Text
                         = "/var/buildkite/shared"
 
+                    let entrypoint
+                        : Text
+                        = if docker.useBash then "/bin/bash" else "/bin/sh"
+
                     in  { line =
-                            "docker run -it --rm --entrypoint /bin/sh --init --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${      if docker.privileged
+                            "docker run -it --rm --entrypoint ${entrypoint} --init --volume /var/secrets:/var/secrets --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${      if docker.privileged
 
-                                                                                                                                                                       then  " --privileged"
+                                                                                                                                                                                                                then  " --privileged"
 
-                                                                                                                                                                       else  ""} ${docker.image} -c '${inner.line}'"
+                                                                                                                                                                                                                else  ""} ${docker.image} -c '${inner.line}'"
                         , readable =
                             Optional/map
                               Text
@@ -130,7 +142,7 @@ let tests =
       let dockerExample =
               assert
             :     { line =
-                      "docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
+                      "docker run -it --rm --entrypoint /bin/bash --init --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'"
                   , readable = Some "Docker@foo/bar:tag ( echo hello )"
                   }
               ===  M.inDocker
@@ -142,7 +154,7 @@ let tests =
 
       let cacheExample =
               assert
-            :     "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
+            :     "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/bash --init --volume /var/secrets:/var/secrets --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\""
               ===  M.format
                      ( M.cacheThrough
                          M.Docker::{

scripts/version-linter.py

@@ -31,19 +31,50 @@ def set_error():
   global exit_code
   exit_code=1
 
-def branch_commit(branch):
+def branch_commits(branch,n):
   print ('Retrieving', branch, 'head commit...')
   result=subprocess.run(['git','log','-n','1','--format="%h"','--abbrev=7',f'{branch}'],
                         capture_output=True)
   output=result.stdout.decode('ascii')
   print ('command stdout:', output)
   print ('command stderr:', result.stderr.decode('ascii'))
-  return output.replace('"','').replace('\n','')
-
-def download_type_shapes(role,branch,sha1) :
+  return output.replace('"','').splitlines()
+
+def url_to_type_shape_file(file):
+  '''
+    Return url to mina type shape file
+  '''
+  return f'https://storage.googleapis.com/mina-type-shapes/{file}'
+
+def sha_exists(sha1):
+  '''
+    Checks if mina type shape with given sha exists
+  '''
+  file = type_shape_file(sha1)
+  return url_exists(url_to_type_shape_file(file))
+
+def url_exists(url):
+  '''
+    Checks if url exists (by sending head and validating that status code is ok)
+  '''
+  return requests.head(url).status_code == 200
+
+def find_latest_type_shape_ref_on(branch,n=1):
+  '''
+    Function tries to find best type shape reference commit by retrieving n last commits 
+    and iterate over collection testing if any item points to valid url   
+  '''
+  commits = branch_commits(branch, n)
+  candidates = list(filter(lambda x: sha_exists(x), commits))
+  if not any(candidates): 
+    raise Exception(f'Cannot find type shape file for {branch}. I tried {n} last commits')
+  else: 
+    return candidates[0]
+
+def download_type_shape(role,branch,sha1) :
   file=type_shape_file(sha1)
   print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1)
-  result=subprocess.run(['wget','--no-clobber',f'https://storage.googleapis.com/mina-type-shapes/{file}'])
+  result=subprocess.run(['wget','--no-clobber',url_to_type_shape_file(file)])
 
 def type_shape_file(sha1) :
   # created by buildkite build-artifact script
@@ -237,18 +268,18 @@ def assert_commit(commit, desc):
 
   subprocess.run(['git','fetch'],capture_output=False)
 
-  base_branch_commit=branch_commit(base_branch)
-  download_type_shapes('base',base_branch,base_branch_commit)
+  base_branch_commit = find_latest_type_shape_ref_on(base_branch,n=10)
+  download_type_shape('base',base_branch,base_branch_commit)
 
   print('')
 
-  release_branch_commit=branch_commit(release_branch)
-  download_type_shapes('release',release_branch,release_branch_commit)
+  release_branch_commit=find_latest_type_shape_ref_on(release_branch, n=10)
+  download_type_shape('release',release_branch,release_branch_commit)
 
   print('')
 
-  pr_branch_commit=branch_commit(pr_branch)
-  download_type_shapes('pr',pr_branch,pr_branch_commit)
+  pr_branch_commit=find_latest_type_shape_ref_on(pr_branch)
+  download_type_shape('pr',pr_branch,pr_branch_commit)
 
   print('')