phase1: raise priority of tag builds

Currently the buildmaster would only order tag builds within their own
branch, meaning that if e.g. a higher priority branch has normal
buildrequests (i.e. buildrequests comming from the AnyBranchScheduler),
and a lower priority branch has "tag" buildrequests (i.e. from the
Triggerable scheduler), the former builderequests would be served first,
deferring the build of e.g. release tags.

We want forced builds (release tag) to have maximum priority, regardless
of branch priority. This commit attempts to solve this problem by
leveraging the newly (as of buildbot 3.9.0) introduced "priority"
scheduler parameter, by raising the Triggerable scheduler buildrequests
priority, and then considering this higher priority in Builders' order.

The net result is that Builders are now prioritized if they have
pending higher priority buildrequest, still preserving the branch order.
In other words, tag requests are front run while preserving branch order,
meaning that if two branches have tag buildrequests, the higher priority
branch is still served first.

This requires buildbot 3.9.0

Signed-off-by: Thibaut VARÈNE <[email protected]>

.github/workflows/build-push.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
 
 env:
-  BUILDBOT_VERSION: 3.8.0
+  BUILDBOT_VERSION: 3.9.0
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

phase1/master.cfg

@@ -191,16 +191,20 @@ c["configurators"] = [
 
 
 @defer.inlineCallbacks
-def getNewestCompleteTime(bldr):
-    """Returns the complete_at of the latest completed and not SKIPPED
+def getNewestCompleteTimePrio(bldr):
+    """Returns the priority and the complete_at of the latest completed and not SKIPPED
     build request for this builder, or None if there are no such build
     requests. We need to filter out SKIPPED requests because we're
     using collapseRequests=True which is unfortunately marking all
     previous requests as complete when new buildset is created.
 
-    @returns: datetime instance or None, via Deferred
+    @returns: (priority, datetime instance or None), via Deferred
     """
 
+    prio = yield bldr.get_highest_priority()
+    if prio is None:
+        prio = 0
+
     bldrid = yield bldr.getBuilderId()
     completed = yield bldr.master.data.get(
         ("builders", bldrid, "buildrequests"),
@@ -212,7 +216,7 @@ def getNewestCompleteTime(bldr):
         limit=1,
     )
     if not completed:
-        return
+        return (prio, None)
 
     complete_at = completed[0]["complete_at"]
 
@@ -228,9 +232,9 @@ def getNewestCompleteTime(bldr):
     if last_build and last_build[0]:
         last_complete_at = last_build[0]["complete_at"]
         if last_complete_at and (last_complete_at > complete_at):
-            return last_complete_at
+            return (prio, last_complete_at)
 
-    return complete_at
+    return (prio, complete_at)
 
 
 @defer.inlineCallbacks
@@ -251,19 +255,23 @@ def prioritizeBuilders(master, builders):
         return bool(bldr.building) or bool(bldr.old_building)
 
     def bldr_info(bldr):
-        d = defer.maybeDeferred(getNewestCompleteTime, bldr)
-        d.addCallback(lambda complete_at: (complete_at, bldr))
+        d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr)
+        d.addCallback(lambda retval: (retval, bldr))
         return d
 
     def bldr_sort(item):
-        (complete_at, bldr) = item
+        ((hiprio, complete_at), bldr) = item
 
+        # check if we have some high prio build requests pending (i.e. tag builds),
+        # if so, front-run these builders, while preserving the per-branch static priority
         pos = 99
         for name, prio in bldrNamePrio.items():
             if bldr.name.startswith(name):
-                pos = prio
+                pos = prio + 50 - min(hiprio, 50)    # higher priority (larger positive number) raises position
                 break
 
+        # pos order: janitor/local (0), tag builds per branch order if any [1..50], !tag builds per branch order [51...]
+
         if not complete_at:
             date = datetime.min
             complete_at = date.replace(tzinfo=tzutc())
@@ -531,7 +539,7 @@ c["schedulers"].append(
 )
 
 c["schedulers"].append(
-    schedulers.Triggerable(name="trigger", builderNames=builderNames)
+    schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20)
 )
 
 ####### BUILDERS