From 52dadfff043d71280ee8db736fdea74604979cb8 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Mon, 15 Apr 2024 16:47:26 +0200 Subject: [PATCH 1/3] feat(ai): add pipelines optimization flags This commit adds a new `OptimizationFlags` field to the `aiModels` config so that users can forward optimization environment variables to the [ai-worker](git@github.com:livepeer/ai-worker.git) for more information see https://github.com/livepeer/ai-worker/pull/61. --- cmd/livepeer/starter/starter.go | 11 ++++++++++- core/ai.go | 17 +++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 79367475b0..1258084c3f 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" ethcommon "github.com/ethereum/go-ethereum/common" @@ -538,6 +539,7 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { return } + var once sync.Once for _, config := range configs { modelConstraint := &core.ModelConstraint{Warm: config.Warm} @@ -545,12 +547,19 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { // the endpoint for an external container if config.Warm || config.URL != "" { endpoint := worker.RunnerEndpoint{URL: config.URL, Token: config.Token} - if err := n.AIWorker.Warm(ctx, config.Pipeline, config.ModelID, endpoint); err != nil { + if err := n.AIWorker.Warm(ctx, config.Pipeline, config.ModelID, endpoint, config.OptimizationFlags); err != nil { glog.Errorf("Error AI worker warming %v container: %v", config.Pipeline, err) return } } + // Show warning if people set OptimizationFlags but not Warm. + if len(config.OptimizationFlags) > 0 && !config.Warm { + once.Do(func() { + glog.Warningf("OptimizationFlags set for model %v but Warm is not set. OptimizationFlags are currently only used for warm containers.", config.ModelID) + }) + } + switch config.Pipeline { case "text-to-image": _, ok := constraints[core.Capability_TextToImage] diff --git a/core/ai.go b/core/ai.go index 384762c3c2..4a7a5ecd0a 100644 --- a/core/ai.go +++ b/core/ai.go @@ -15,18 +15,19 @@ type AI interface { TextToImage(context.Context, worker.TextToImageJSONRequestBody) (*worker.ImageResponse, error) ImageToImage(context.Context, worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) ImageToVideo(context.Context, worker.ImageToVideoMultipartRequestBody) (*worker.VideoResponse, error) - Warm(context.Context, string, string, worker.RunnerEndpoint) error + Warm(context.Context, string, string, worker.RunnerEndpoint, worker.OptimizationFlags) error Stop(context.Context) error } type AIModelConfig struct { - Pipeline string `json:"pipeline"` - ModelID string `json:"model_id"` - URL string `json:"url,omitempty"` - Token string `json:"token,omitempty"` - Warm bool `json:"warm,omitempty"` - PricePerUnit int64 `json:"price_per_unit,omitempty"` - PixelsPerUnit int64 `json:"pixels_per_unit,omitempty"` + Pipeline string `json:"pipeline"` + ModelID string `json:"model_id"` + URL string `json:"url,omitempty"` + Token string `json:"token,omitempty"` + Warm bool `json:"warm,omitempty"` + PricePerUnit int64 `json:"price_per_unit,omitempty"` + PixelsPerUnit int64 `json:"pixels_per_unit,omitempty"` + OptimizationFlags worker.OptimizationFlags `json:"optimization_flags,omitempty"` } func (config *AIModelConfig) UnmarshalJSON(data []byte) error { From bb50339b2bf262458a203ac2160f7ad61782c7bf Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Tue, 16 Apr 2024 12:30:17 +0200 Subject: [PATCH 2/3] chore: update ai-worker to latest commit This commit ensures that the https://github.com/livepeer/ai-worker dependency is on the latest commit that includes the new optimization flags feature (see https://github.com/livepeer/ai-worker/pull/61). --- go.mod | 4 ++-- go.sum | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 3ef60906c9..339f8805ac 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/jaypipes/ghw v0.10.0 github.com/jaypipes/pcidb v1.0.0 - github.com/livepeer/ai-worker v0.0.0-20240220213200-59b5b237cd8b + github.com/livepeer/ai-worker v0.0.0-20240416102817-db751cd47fdc github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 @@ -32,6 +32,7 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/net v0.19.0 google.golang.org/grpc v1.57.1 + google.golang.org/protobuf v1.31.0 pgregory.net/rapid v1.1.0 ) @@ -197,7 +198,6 @@ require ( google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index d4b9a93d3d..a19f36cfb2 100644 --- a/go.sum +++ b/go.sum @@ -532,14 +532,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= -github.com/livepeer/ai-worker v0.0.0-20240213170524-1c860ab6e412 h1:zc6XbU0KvJ8C2jDdsZP8IkWeKN0TCQMIUyl3fOEydoU= -github.com/livepeer/ai-worker v0.0.0-20240213170524-1c860ab6e412/go.mod h1:JvUlcQktSgkEfzotuelfw9OpjGi2qZTW/3tWB/klb/c= -github.com/livepeer/ai-worker v0.0.0-20240214164547-fbdca26a9b2d h1:YpBW6wqwpQ9ffgXPnCEoMbsQEbCT5G2AbTb91jrD6BU= -github.com/livepeer/ai-worker v0.0.0-20240214164547-fbdca26a9b2d/go.mod h1:JvUlcQktSgkEfzotuelfw9OpjGi2qZTW/3tWB/klb/c= -github.com/livepeer/ai-worker v0.0.0-20240214223314-3d355743417a h1:LZwlUatQLU7rkICGMmca71DE67iGhln/QttAehEs8LA= -github.com/livepeer/ai-worker v0.0.0-20240214223314-3d355743417a/go.mod h1:JvUlcQktSgkEfzotuelfw9OpjGi2qZTW/3tWB/klb/c= -github.com/livepeer/ai-worker v0.0.0-20240220213200-59b5b237cd8b h1:zCQv/A3Pafdr/NutU7zLChkF+XcAdzY0ObjN1E4nr44= -github.com/livepeer/ai-worker v0.0.0-20240220213200-59b5b237cd8b/go.mod h1:JvUlcQktSgkEfzotuelfw9OpjGi2qZTW/3tWB/klb/c= +github.com/livepeer/ai-worker v0.0.0-20240416102817-db751cd47fdc h1:wmXT+pkcAshQ5yYiDLpoeIY0+P3Ob4RxHOljbS13FfA= +github.com/livepeer/ai-worker v0.0.0-20240416102817-db751cd47fdc/go.mod h1:JvUlcQktSgkEfzotuelfw9OpjGi2qZTW/3tWB/klb/c= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA= From fd3397a32b820b206f876ff414cf5a9807efc7b2 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Tue, 16 Apr 2024 15:38:15 +0200 Subject: [PATCH 3/3] refactor: improve OptFlags logging This commit ensures that the `optimzation flag not supported` warning is shown for each model that is not loaded warm. --- cmd/livepeer/starter/starter.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 1258084c3f..590edaebdb 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -18,7 +18,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" ethcommon "github.com/ethereum/go-ethereum/common" @@ -539,7 +538,6 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { return } - var once sync.Once for _, config := range configs { modelConstraint := &core.ModelConstraint{Warm: config.Warm} @@ -555,9 +553,7 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { // Show warning if people set OptimizationFlags but not Warm. if len(config.OptimizationFlags) > 0 && !config.Warm { - once.Do(func() { - glog.Warningf("OptimizationFlags set for model %v but Warm is not set. OptimizationFlags are currently only used for warm containers.", config.ModelID) - }) + glog.Warningf("Model %v has 'optimization_flags' set without 'warm'. Optimization flags are currently only used for warm containers.", config.ModelID) } switch config.Pipeline {