From 6fcc75d9c225212af759972f65285b727b7ec999 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 12 Jun 2024 14:48:18 -0400 Subject: [PATCH 01/34] Add speech-to-text pipeline, refactor processAIRequest and handleAIRequest to allow for various response types --- cmd/livepeer/starter/starter.go | 13 +++ core/ai.go | 1 + core/capabilities.go | 3 + core/orchestrator.go | 16 ++++ server/ai_http.go | 57 +++++++++++-- server/ai_mediaserver.go | 50 +++++++++++ server/ai_process.go | 141 +++++++++++++++++++++++++------- server/rpc.go | 1 + 8 files changed, 249 insertions(+), 33 deletions(-) diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 15cca7f8c3..34e09fa232 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -613,6 +613,19 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { constraints[core.Capability_Upscale].Models[config.ModelID] = modelConstraint n.SetBasePriceForCap("default", core.Capability_Upscale, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) + case "speech-to-text": + _, ok := constraints[core.Capability_SpeechToText] + if !ok { + aiCaps = append(aiCaps, core.Capability_SpeechToText) + constraints[core.Capability_SpeechToText] = &core.Constraints{ + Models: make(map[string]*core.ModelConstraint), + } + } + + constraints[core.Capability_SpeechToText].Models[config.ModelID] = modelConstraint + + n.SetBasePriceForCap("default", core.Capability_SpeechToText, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) + } if len(aiCaps) > 0 { diff --git a/core/ai.go b/core/ai.go index c4a7146ec6..9a8807ff44 100644 --- a/core/ai.go +++ b/core/ai.go @@ -16,6 +16,7 @@ type AI interface { ImageToImage(context.Context, worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) ImageToVideo(context.Context, worker.ImageToVideoMultipartRequestBody) (*worker.VideoResponse, error) Upscale(context.Context, worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error) + SpeechToText(context.Context, worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) Warm(context.Context, string, string, worker.RunnerEndpoint, worker.OptimizationFlags) error Stop(context.Context) error HasCapacity(pipeline, modelID string) bool diff --git a/core/capabilities.go b/core/capabilities.go index 240222ab29..3615a30711 100644 --- a/core/capabilities.go +++ b/core/capabilities.go @@ -71,6 +71,7 @@ const ( Capability_ImageToImage Capability_ImageToVideo Capability_Upscale + Capability_SpeechToText ) var CapabilityNameLookup = map[Capability]string{ @@ -106,6 +107,7 @@ var CapabilityNameLookup = map[Capability]string{ Capability_ImageToImage: "Image to image", Capability_ImageToVideo: "Image to video", Capability_Upscale: "Upscale", + Capability_SpeechToText: "Speech to text", } var CapabilityTestLookup = map[Capability]CapabilityTest{ @@ -195,6 +197,7 @@ func OptionalCapabilities() []Capability { Capability_ImageToImage, Capability_ImageToVideo, Capability_Upscale, + Capability_SpeechToText, } } diff --git a/core/orchestrator.go b/core/orchestrator.go index f9d42b3faf..a29be55f52 100644 --- a/core/orchestrator.go +++ b/core/orchestrator.go @@ -126,6 +126,10 @@ func (orch *orchestrator) Upscale(ctx context.Context, req worker.UpscaleMultipa return orch.node.upscale(ctx, req) } +func (orch *orchestrator) SpeechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { + return orch.node.speechToText(ctx, req) +} + func (orch *orchestrator) ProcessPayment(ctx context.Context, payment net.Payment, manifestID ManifestID) error { if orch.node == nil || orch.node.Recipient == nil { return nil @@ -945,6 +949,18 @@ func (n *LivepeerNode) upscale(ctx context.Context, req worker.UpscaleMultipartR return n.AIWorker.Upscale(ctx, req) } +func (n *LivepeerNode) speechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { + + resp, err := n.AIWorker.SpeechToText(ctx, req) + if err != nil { + return nil, err + } + + return resp, nil + //return &worker.ImageResponse{Images: videos}, nil + +} + func (n *LivepeerNode) imageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) { // We might support generating more than one video in the future (i.e. multiple input images/prompts) numVideos := 1 diff --git a/server/ai_http.go b/server/ai_http.go index fea249ca51..81db815d7b 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -42,6 +42,7 @@ func startAIServer(lp lphttp) error { lp.transRPC.Handle("/image-to-image", oapiReqValidator(lp.ImageToImage())) lp.transRPC.Handle("/image-to-video", oapiReqValidator(lp.ImageToVideo())) lp.transRPC.Handle("/upscale", oapiReqValidator(lp.Upscale())) + lp.transRPC.Handle("/speech-to-text", oapiReqValidator(lp.SpeechToText())) return nil } @@ -132,6 +133,29 @@ func (h *lphttp) Upscale() http.Handler { }) } +func (h *lphttp) SpeechToText() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + orch := h.orchestrator + + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + + multiRdr, err := r.MultipartReader() + if err != nil { + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + + var req worker.SpeechToTextMultipartRequestBody + if err := runtime.BindMultipart(&req, *multiRdr); err != nil { + respondWithError(w, err.Error(), http.StatusInternalServerError) + return + } + + handleAIRequest(ctx, w, r, orch, req) + }) +} + func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, orch Orchestrator, req interface{}) { payment, err := getPayment(r.Header.Get(paymentHeader)) if err != nil { @@ -149,7 +173,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request var cap core.Capability var pipeline string var modelID string - var submitFn func(context.Context) (*worker.ImageResponse, error) + var submitFn func(context.Context) (interface{}, error) var outPixels int64 switch v := req.(type) { @@ -157,7 +181,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request pipeline = "text-to-image" cap = core.Capability_TextToImage modelID = *v.ModelId - submitFn = func(ctx context.Context) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context) (interface{}, error) { return orch.TextToImage(ctx, v) } @@ -176,7 +200,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request pipeline = "image-to-image" cap = core.Capability_ImageToImage modelID = *v.ModelId - submitFn = func(ctx context.Context) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context) (interface{}, error) { return orch.ImageToImage(ctx, v) } @@ -195,7 +219,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request pipeline = "upscale" cap = core.Capability_Upscale modelID = *v.ModelId - submitFn = func(ctx context.Context) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context) (interface{}, error) { return orch.Upscale(ctx, v) } @@ -214,7 +238,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request pipeline = "image-to-video" cap = core.Capability_ImageToVideo modelID = *v.ModelId - submitFn = func(ctx context.Context) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context) (interface{}, error) { return orch.ImageToVideo(ctx, v) } @@ -231,6 +255,29 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request frames := int64(25) outPixels = height * width * int64(frames) + case worker.SpeechToTextMultipartRequestBody: + pipeline = "speech-to-text" + cap = core.Capability_SpeechToText + modelID = *v.ModelId + submitFn = func(ctx context.Context) (interface{}, error) { + return orch.SpeechToText(ctx, v) + } + + //imageRdr, err := v.Audio.Reader() + //if err != nil { + // respondWithError(w, err.Error(), http.StatusBadRequest) + // return + //} + + //config, _, err := image.DecodeConfig(imageRdr) + //if err != nil { + // respondWithError(w, err.Error(), http.StatusBadRequest) + // return + //} + + //TODO: Calculate outPixels based on audio duration + outPixels = 50 + default: respondWithError(w, "Unknown request type", http.StatusBadRequest) return diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index ee58b340f0..a0d4665080 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -68,6 +68,7 @@ func startAIMediaServer(ls *LivepeerServer) error { ls.HTTPMux.Handle("/upscale", oapiReqValidator(ls.Upscale())) ls.HTTPMux.Handle("/image-to-video", oapiReqValidator(ls.ImageToVideo())) ls.HTTPMux.Handle("/image-to-video/result", ls.ImageToVideoResult()) + ls.HTTPMux.Handle("/speech-to-text", oapiReqValidator(ls.SpeechToText())) return nil } @@ -320,6 +321,55 @@ func (ls *LivepeerServer) Upscale() http.Handler { }) } +func (ls *LivepeerServer) SpeechToText() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "request_id", requestID) + + multiRdr, err := r.MultipartReader() + if err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } + + var req worker.SpeechToTextMultipartRequestBody + if err := runtime.BindMultipart(&req, *multiRdr); err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } + + //TODO: add duration to log here + clog.V(common.VERBOSE).Infof(ctx, "Received SpeechToText request model_id=%v", *req.ModelId) + + params := aiRequestParams{ + node: ls.LivepeerNode, + os: drivers.NodeStorage.NewSession(string(core.RandomManifestID())), + sessManager: ls.AISessionManager, + } + + start := time.Now() + resp, err := processSpeechToText(ctx, params, req) + if err != nil { + var e *ServiceUnavailableError + if errors.As(err, &e) { + respondJsonError(ctx, w, err, http.StatusServiceUnavailable) + return + } + respondJsonError(ctx, w, err, http.StatusInternalServerError) + return + } + + took := time.Since(start) + clog.V(common.VERBOSE).Infof(ctx, "Processed SpeechToText request model_id=%v took=%v", *req.ModelId, took) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) + }) +} + func (ls *LivepeerServer) ImageToVideoResult() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) diff --git a/server/ai_process.go b/server/ai_process.go index 697943304b..bdf4377adf 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -28,6 +28,7 @@ const defaultTextToImageModelID = "stabilityai/sdxl-turbo" const defaultImageToImageModelID = "stabilityai/sdxl-turbo" const defaultImageToVideoModelID = "stabilityai/stable-video-diffusion-img2vid-xt" const defaultUpscaleModelID = "stabilityai/stable-diffusion-x4-upscaler" +const defaultSpeechToTextModelID = "openai/whisper-large-v3" type ServiceUnavailableError struct { err error @@ -49,8 +50,10 @@ func processTextToImage(ctx context.Context, params aiRequestParams, req worker. return nil, err } - newMedia := make([]worker.Media, len(resp.Images)) - for i, media := range resp.Images { + imgResp := resp.(*worker.ImageResponse) + + newMedia := make([]worker.Media, len(imgResp.Images)) + for i, media := range imgResp.Images { var data bytes.Buffer writer := bufio.NewWriter(&data) if err := worker.ReadImageB64DataUrl(media.Url, writer); err != nil { @@ -67,9 +70,9 @@ func processTextToImage(ctx context.Context, params aiRequestParams, req worker. newMedia[i] = worker.Media{Nsfw: media.Nsfw, Seed: media.Seed, Url: newUrl} } - resp.Images = newMedia + imgResp.Images = newMedia - return resp, nil + return imgResp, nil } func submitTextToImage(ctx context.Context, params aiRequestParams, sess *AISession, req worker.TextToImageJSONRequestBody) (*worker.ImageResponse, error) { @@ -133,8 +136,10 @@ func processImageToImage(ctx context.Context, params aiRequestParams, req worker return nil, err } - newMedia := make([]worker.Media, len(resp.Images)) - for i, media := range resp.Images { + imgResp := resp.(*worker.ImageResponse) + + newMedia := make([]worker.Media, len(imgResp.Images)) + for i, media := range imgResp.Images { var data bytes.Buffer writer := bufio.NewWriter(&data) if err := worker.ReadImageB64DataUrl(media.Url, writer); err != nil { @@ -151,9 +156,9 @@ func processImageToImage(ctx context.Context, params aiRequestParams, req worker newMedia[i] = worker.Media{Nsfw: media.Nsfw, Seed: media.Seed, Url: newUrl} } - resp.Images = newMedia + imgResp.Images = newMedia - return resp, nil + return imgResp, nil } func submitImageToImage(ctx context.Context, params aiRequestParams, sess *AISession, req worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) { @@ -214,8 +219,11 @@ func processImageToVideo(ctx context.Context, params aiRequestParams, req worker } // HACK: Re-use worker.ImageResponse to return results - videos := make([]worker.Media, len(resp.Images)) - for i, media := range resp.Images { + // TODO: Refactor to return worker.VideoResponse + imgResp := resp.(*worker.ImageResponse) + + videos := make([]worker.Media, len(imgResp.Images)) + for i, media := range imgResp.Images { data, err := downloadSeg(ctx, media.Url) if err != nil { return nil, err @@ -235,9 +243,9 @@ func processImageToVideo(ctx context.Context, params aiRequestParams, req worker } - resp.Images = videos + imgResp.Images = videos - return resp, nil + return imgResp, nil } func submitImageToVideo(ctx context.Context, params aiRequestParams, sess *AISession, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) { @@ -308,8 +316,10 @@ func processUpscale(ctx context.Context, params aiRequestParams, req worker.Upsc return nil, err } - newMedia := make([]worker.Media, len(resp.Images)) - for i, media := range resp.Images { + imgResp := resp.(*worker.ImageResponse) + + newMedia := make([]worker.Media, len(imgResp.Images)) + for i, media := range imgResp.Images { var data bytes.Buffer writer := bufio.NewWriter(&data) if err := worker.ReadImageB64DataUrl(media.Url, writer); err != nil { @@ -326,9 +336,9 @@ func processUpscale(ctx context.Context, params aiRequestParams, req worker.Upsc newMedia[i] = worker.Media{Nsfw: media.Nsfw, Seed: media.Seed, Url: newUrl} } - resp.Images = newMedia + imgResp.Images = newMedia - return resp, nil + return imgResp, nil } func submitUpscale(ctx context.Context, params aiRequestParams, sess *AISession, req worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error) { @@ -382,10 +392,76 @@ func submitUpscale(ctx context.Context, params aiRequestParams, sess *AISession, return resp.JSON200, nil } -func processAIRequest(ctx context.Context, params aiRequestParams, req interface{}) (*worker.ImageResponse, error) { +func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISession, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { + var buf bytes.Buffer + mw, err := worker.NewSpeechToTextMultipartWriter(&buf, req) + if err != nil { + return nil, err + } + + client, err := worker.NewClientWithResponses(sess.Transcoder(), worker.WithHTTPClient(httpClient)) + if err != nil { + return nil, err + } + + // //TODO: Setting static for now until pipeline is functional (50 pixels) + outPixels := int64(50) + // //TODO: Measure length of auto and implement pricing unit + + setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) + if err != nil { + return nil, err + } + defer completeBalanceUpdate(sess.BroadcastSession, balUpdate) + + start := time.Now() + resp, err := client.SpeechToTextWithBody(ctx, mw.FormDataContentType(), &buf, setHeaders) + took := time.Since(start) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, errors.New(string(data)) + } + + // We treat a response as "receiving change" where the change is the difference between the credit and debit for the update + if balUpdate != nil { + balUpdate.Status = ReceivedChange + } + + var res worker.TextResponse + if err := json.Unmarshal(data, &res); err != nil { + return nil, err + } + + // TODO: Refine this rough estimate in future iterations + sess.LatencyScore = took.Seconds() / float64(outPixels) + + return &res, nil +} + +func processSpeechToText(ctx context.Context, params aiRequestParams, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { + resp, err := processAIRequest(ctx, params, req) + if err != nil { + return nil, err + } + + txtResp := resp.(*worker.TextResponse) + + return txtResp, nil +} + +func processAIRequest(ctx context.Context, params aiRequestParams, req interface{}) (interface{}, error) { var cap core.Capability var modelID string - var submitFn func(context.Context, aiRequestParams, *AISession) (*worker.ImageResponse, error) + var submitFn func(context.Context, aiRequestParams, *AISession) (interface{}, error) switch v := req.(type) { case worker.TextToImageJSONRequestBody: @@ -394,7 +470,7 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface if v.ModelId != nil { modelID = *v.ModelId } - submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitTextToImage(ctx, params, sess, v) } case worker.ImageToImageMultipartRequestBody: @@ -403,7 +479,7 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface if v.ModelId != nil { modelID = *v.ModelId } - submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitImageToImage(ctx, params, sess, v) } case worker.ImageToVideoMultipartRequestBody: @@ -412,7 +488,8 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface if v.ModelId != nil { modelID = *v.ModelId } - submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (*worker.ImageResponse, error) { + // Assuming submitImageToVideo returns a VideoResponse + submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitImageToVideo(ctx, params, sess, v) } case worker.UpscaleMultipartRequestBody: @@ -421,14 +498,25 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface if v.ModelId != nil { modelID = *v.ModelId } - submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (*worker.ImageResponse, error) { + submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitUpscale(ctx, params, sess, v) } + case worker.SpeechToTextMultipartRequestBody: + cap = core.Capability_SpeechToText + modelID = defaultSpeechToTextModelID + if v.ModelId != nil { + modelID = *v.ModelId + } + submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { + return submitSpeechToText(ctx, params, sess, v) + } + // Add more cases as needed... default: - return nil, errors.New("unknown AI request type") + return nil, fmt.Errorf("unsupported request type %T", req) } - var resp *worker.ImageResponse + //var resp *worker.ImageResponse + var resp interface{} cctx, cancel := context.WithTimeout(ctx, processingRetryTimeout) defer cancel() @@ -457,17 +545,14 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface params.sessManager.Complete(ctx, sess) break } - clog.Infof(ctx, "Error submitting request cap=%v modelID=%v try=%v orch=%v err=%v", cap, modelID, tries, sess.Transcoder(), err) - params.sessManager.Remove(ctx, sess) } if resp == nil { return nil, &ServiceUnavailableError{err: errors.New("no orchestrators available")} } - - return resp, nil + return resp.(interface{}), nil } func prepareAIPayment(ctx context.Context, sess *AISession, outPixels int64) (worker.RequestEditorFn, *BalanceUpdate, error) { diff --git a/server/rpc.go b/server/rpc.go index 9c24d3336a..ff5b21ac6d 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -67,6 +67,7 @@ type Orchestrator interface { ImageToImage(ctx context.Context, req worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) ImageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) Upscale(ctx context.Context, req worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error) + SpeechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) } // Balance describes methods for a session's balance maintenance From d4ba500dda02431f3627d7979d01e299c071bdec Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 12 Jun 2024 15:19:25 -0400 Subject: [PATCH 02/34] Pin gomod to ai-runner for testing --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e77da1df23..852a3ca417 100644 --- a/go.mod +++ b/go.mod @@ -205,3 +205,5 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd diff --git a/go.sum b/go.sum index 3073624a39..655500050b 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd h1:iW2L7BmGPyyF1lgxFF1Z2zKKGNrxVchy74bPTm1Fv7g= +github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -531,8 +533,6 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= -github.com/livepeer/ai-worker v0.0.8 h1:FAjYJgSOaZslA06Wb6MolYohI30IMIujDTB26nfw8YE= -github.com/livepeer/ai-worker v0.0.8/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA= From b0fa4b760f42c082fd5d91d7b6b16ab0bf0d13d0 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 12 Jun 2024 15:26:54 -0400 Subject: [PATCH 03/34] Revert "Pin gomod to ai-runner for testing" This reverts commit d4ba500dda02431f3627d7979d01e299c071bdec. --- go.mod | 2 -- go.sum | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 852a3ca417..e77da1df23 100644 --- a/go.mod +++ b/go.mod @@ -205,5 +205,3 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd diff --git a/go.sum b/go.sum index 655500050b..3073624a39 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,6 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd h1:iW2L7BmGPyyF1lgxFF1Z2zKKGNrxVchy74bPTm1Fv7g= -github.com/eliteprox/ai-worker v0.0.0-20240611132628-0153d6a879bd/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -533,6 +531,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= +github.com/livepeer/ai-worker v0.0.8 h1:FAjYJgSOaZslA06Wb6MolYohI30IMIujDTB26nfw8YE= +github.com/livepeer/ai-worker v0.0.8/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA= From 2144538eb230482aa1b7830e409e219344772aab Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 12 Jun 2024 15:32:18 -0400 Subject: [PATCH 04/34] Update go mod dep for ai-worker --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e77da1df23..8142de8e6a 100644 --- a/go.mod +++ b/go.mod @@ -205,3 +205,5 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1 diff --git a/go.sum b/go.sum index 3073624a39..01c0070dc1 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1 h1:Ao8wONCNgzTpOC79upkZXMf2iSmjFr0Iejb4xHgvZkE= +github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -531,8 +533,6 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= -github.com/livepeer/ai-worker v0.0.8 h1:FAjYJgSOaZslA06Wb6MolYohI30IMIujDTB26nfw8YE= -github.com/livepeer/ai-worker v0.0.8/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA= From 8d424ff385ee6c93dea690828940e96f42aaf6a4 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Sun, 16 Jun 2024 03:27:05 -0400 Subject: [PATCH 05/34] Calculate pixel value of audio file --- server/ai_http.go | 51 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/server/ai_http.go b/server/ai_http.go index 81db815d7b..677f02d568 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "image" + "io" "net/http" "strconv" "time" @@ -16,6 +17,8 @@ import ( "github.com/livepeer/go-livepeer/core" middleware "github.com/oapi-codegen/nethttp-middleware" "github.com/oapi-codegen/runtime" + "github.com/oapi-codegen/runtime/types" + "github.com/tcolgate/mp3" ) func startAIServer(lp lphttp) error { @@ -263,20 +266,35 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request return orch.SpeechToText(ctx, v) } - //imageRdr, err := v.Audio.Reader() - //if err != nil { - // respondWithError(w, err.Error(), http.StatusBadRequest) - // return - //} - - //config, _, err := image.DecodeConfig(imageRdr) - //if err != nil { - // respondWithError(w, err.Error(), http.StatusBadRequest) - // return - //} - - //TODO: Calculate outPixels based on audio duration - outPixels = 50 + audio, err := v.Audio.Reader() + if err != nil { + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + decoder := mp3.NewDecoder(audio) + defer func() { + if r := recover(); r != nil { + audio.Close() + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + }() + totalDuration := 0.0 + var frame mp3.Frame + var skipped int + var frameCount int + for { + if err := decoder.Decode(&frame, &skipped); err != nil { + if err == io.EOF { + break + } + return + } + totalDuration += frame.Duration().Seconds() * (float64(frame.Header().BitRate() / 10)) + frameCount++ + } + audio.Close() + outPixels = int64(totalDuration) default: respondWithError(w, "Unknown request type", http.StatusBadRequest) @@ -334,3 +352,8 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(resp) } + +func getAudioDuration(file types.File) { + + panic("unimplemented") +} From 4d7674944292470b7709b863e02ff65ecf28a3c5 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Sun, 16 Jun 2024 15:15:36 -0400 Subject: [PATCH 06/34] fix go-mod deps --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index 8142de8e6a..7dd25778e3 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( pgregory.net/rapid v1.1.0 ) +require github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 + require ( cloud.google.com/go v0.110.2 // indirect cloud.google.com/go/compute v1.20.0 // indirect diff --git a/go.sum b/go.sum index 01c0070dc1..40558b8f6c 100644 --- a/go.sum +++ b/go.sum @@ -788,6 +788,8 @@ github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 h1:XQdibLKagjdevRB6vAjVY4qbSr8rQ610YzTkWcxzxSI= +github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300/go.mod h1:FNa/dfN95vAYCNFrIKRrlRo+MBLbwmR9Asa5f2ljmBI= github.com/testcontainers/testcontainers-go v0.9.0 h1:ZyftCfROjGrKlxk3MOUn2DAzWrUtzY/mj17iAkdUIvI= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= From 1104708c72e91370fdf3d015a6dc6b9ac655bf61 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 19 Jun 2024 00:41:19 -0400 Subject: [PATCH 07/34] Adjust price calculation --- server/ai_http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/ai_http.go b/server/ai_http.go index 677f02d568..d860fa9027 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -290,11 +290,11 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request } return } - totalDuration += frame.Duration().Seconds() * (float64(frame.Header().BitRate() / 10)) + totalDuration += frame.Duration().Seconds() frameCount++ } audio.Close() - outPixels = int64(totalDuration) + outPixels = int64(totalDuration * 1024) default: respondWithError(w, "Unknown request type", http.StatusBadRequest) From 4fcca571a754ae247766e362f8f1a7a848c7dc34 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 19 Jun 2024 01:05:05 -0400 Subject: [PATCH 08/34] one second per pixel --- server/ai_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ai_http.go b/server/ai_http.go index d860fa9027..6d7db25af6 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -294,7 +294,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request frameCount++ } audio.Close() - outPixels = int64(totalDuration * 1024) + outPixels = int64(totalDuration) default: respondWithError(w, "Unknown request type", http.StatusBadRequest) From 02802965fc1dc525c77d03c57634bdf1fd7f10b3 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 19 Jun 2024 05:06:31 -0400 Subject: [PATCH 09/34] cleanup, fix missing duration --- common/util.go | 27 +++++++++++++++++++++++++++ core/orchestrator.go | 2 -- server/ai_http.go | 36 +++--------------------------------- server/ai_mediaserver.go | 1 - server/ai_process.go | 11 ++++++++--- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/common/util.go b/common/util.go index 639cf07ccd..7a666017a6 100644 --- a/common/util.go +++ b/common/util.go @@ -26,6 +26,7 @@ import ( "github.com/livepeer/go-livepeer/net" ffmpeg "github.com/livepeer/lpms/ffmpeg" "github.com/pkg/errors" + "github.com/tcolgate/mp3" "google.golang.org/grpc/peer" ) @@ -530,3 +531,29 @@ func ParseEthAddr(strJsonKey string) (string, error) { } return "", errors.New("Error parsing address from keyfile") } + +// determines the duration of an mp3 audio file by reading the frames +func CalculateAudioDuration(audio io.ReadCloser) (int64, error) { + decoder := mp3.NewDecoder(audio) + defer func() { + if r := recover(); r != nil { + audio.Close() + return + } + }() + totalDuration := 0.0 + var frame mp3.Frame + var skipped int + var frameCount int + for { + if err := decoder.Decode(&frame, &skipped); err != nil { + if err == io.EOF { + break + } + return 0, err + } + totalDuration += frame.Duration().Seconds() + frameCount++ + } + return int64(totalDuration), nil +} diff --git a/core/orchestrator.go b/core/orchestrator.go index a29be55f52..6086067b70 100644 --- a/core/orchestrator.go +++ b/core/orchestrator.go @@ -957,8 +957,6 @@ func (n *LivepeerNode) speechToText(ctx context.Context, req worker.SpeechToText } return resp, nil - //return &worker.ImageResponse{Images: videos}, nil - } func (n *LivepeerNode) imageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) { diff --git a/server/ai_http.go b/server/ai_http.go index 6d7db25af6..3c8e4d97d3 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "image" - "io" "net/http" "strconv" "time" @@ -17,8 +16,6 @@ import ( "github.com/livepeer/go-livepeer/core" middleware "github.com/oapi-codegen/nethttp-middleware" "github.com/oapi-codegen/runtime" - "github.com/oapi-codegen/runtime/types" - "github.com/tcolgate/mp3" ) func startAIServer(lp lphttp) error { @@ -269,33 +266,11 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request audio, err := v.Audio.Reader() if err != nil { respondWithError(w, err.Error(), http.StatusBadRequest) - return } - decoder := mp3.NewDecoder(audio) - defer func() { - if r := recover(); r != nil { - audio.Close() - respondWithError(w, err.Error(), http.StatusBadRequest) - return - } - }() - totalDuration := 0.0 - var frame mp3.Frame - var skipped int - var frameCount int - for { - if err := decoder.Decode(&frame, &skipped); err != nil { - if err == io.EOF { - break - } - return - } - totalDuration += frame.Duration().Seconds() - frameCount++ + outPixels, err = common.CalculateAudioDuration(audio) + if err != nil { + respondWithError(w, "Unable to calculate duration", http.StatusBadRequest) } - audio.Close() - outPixels = int64(totalDuration) - default: respondWithError(w, "Unknown request type", http.StatusBadRequest) return @@ -352,8 +327,3 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(resp) } - -func getAudioDuration(file types.File) { - - panic("unimplemented") -} diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index a0d4665080..895c919fc6 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -340,7 +340,6 @@ func (ls *LivepeerServer) SpeechToText() http.Handler { return } - //TODO: add duration to log here clog.V(common.VERBOSE).Infof(ctx, "Received SpeechToText request model_id=%v", *req.ModelId) params := aiRequestParams{ diff --git a/server/ai_process.go b/server/ai_process.go index bdf4377adf..2d6c1cae00 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -404,10 +404,15 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes return nil, err } - // //TODO: Setting static for now until pipeline is functional (50 pixels) - outPixels := int64(50) - // //TODO: Measure length of auto and implement pricing unit + audio, err := req.Audio.Reader() + if err != nil { + return nil, err + } + outPixels, err := common.CalculateAudioDuration(audio) + if err != nil { + return nil, err + } setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) if err != nil { return nil, err From 34f9d2e2c3d30458a5a78a1a9ba6fb5311bbf0f7 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Thu, 27 Jun 2024 19:56:37 -0400 Subject: [PATCH 10/34] Add supported file types, calculate price by milliseconds --- common/util.go | 109 ++++++++++++++++++++++++++++++++++++++++++- go.mod | 15 +++++- go.sum | 43 +++++++++++++++-- server/ai_http.go | 8 ++-- server/ai_process.go | 8 +--- 5 files changed, 165 insertions(+), 18 deletions(-) diff --git a/common/util.go b/common/util.go index 7a666017a6..65a6930e6c 100644 --- a/common/util.go +++ b/common/util.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "context" "encoding/hex" "encoding/json" @@ -11,6 +12,7 @@ import ( "math/big" "math/rand" "mime" + "path/filepath" "regexp" "sort" "strconv" @@ -18,13 +20,18 @@ import ( "testing" "time" + "github.com/abema/go-mp4" + "github.com/ebml-go/webm" "github.com/ethereum/go-ethereum/crypto" + "github.com/go-audio/wav" "github.com/golang/glog" "github.com/jaypipes/ghw" "github.com/jaypipes/ghw/pkg/gpu" "github.com/jaypipes/ghw/pkg/pci" "github.com/livepeer/go-livepeer/net" ffmpeg "github.com/livepeer/lpms/ffmpeg" + "github.com/mewkiz/flac" + "github.com/oapi-codegen/runtime/types" "github.com/pkg/errors" "github.com/tcolgate/mp3" "google.golang.org/grpc/peer" @@ -533,7 +540,96 @@ func ParseEthAddr(strJsonKey string) (string, error) { } // determines the duration of an mp3 audio file by reading the frames -func CalculateAudioDuration(audio io.ReadCloser) (int64, error) { +func EstimateAudioDuration(audio types.File) (int64, error) { + read, err := audio.Reader() + if err != nil { + return 0, err + } + defer read.Close() + + fileExt := filepath.Ext(audio.Filename()) + switch fileExt { + case ".wav": + return GetDuration_WAV(read) + case ".mp4": + return GetDuration_MP4(read) + case ".m4a": + return GetDuration_MP4(read) + case ".webm": + return GetDuration_WEBM(read) + case ".flac": + return GetDuration_FLAC(read) + case ".mp3": + return GetDuration_MP3(read) + + default: + return 0, errors.New("unsupported audio format") + } +} + +var ErrorCalculatingDuration = errors.New("Error calculating duration") + +func GetDuration_WAV(audio io.ReadCloser) (int64, error) { + seeker, err := GetReadSeeker(audio) + if err != nil { + return 0, ErrorCalculatingDuration + } + + decoder := wav.NewDecoder(seeker) + decoder.ReadInfo() + duration, err := decoder.Duration() + if err != nil { + return 0, ErrorCalculatingDuration + } + + return int64(duration.Seconds()), nil +} + +func GetDuration_MP4(audio io.ReadCloser) (int64, error) { + seeker, err := GetReadSeeker(audio) + if err != nil { + return 0, ErrorCalculatingDuration + } + + path := mp4.BoxPath{mp4.BoxTypeMoov(), mp4.BoxTypeMvhd()} + boxes, err := mp4.ExtractBoxWithPayload(seeker, nil, path) + for _, box := range boxes { + if box.Info.Type == mp4.BoxTypeMvhd() { + if mvhdBox, ok := box.Payload.(*mp4.Mvhd); ok { + durationInSeconds := float64(mvhdBox.GetDuration()) / float64(mvhdBox.Timescale) + return int64(durationInSeconds), nil + } + } + } + + return 0, ErrorCalculatingDuration +} + +func GetDuration_WEBM(audio io.ReadCloser) (int64, error) { + seeker, err := GetReadSeeker(audio) + if err != nil { + return 0, ErrorCalculatingDuration + } + + var webmData webm.WebM + if _, err := webm.Parse(seeker, &webmData); err != nil { + return 0, ErrorCalculatingDuration + } + + return int64(webmData.GetDuration().Seconds()), nil +} + +func GetDuration_FLAC(audio io.ReadCloser) (int64, error) { + stream, err := flac.Parse(audio) + if err != nil { + return 0, ErrorCalculatingDuration + } + + durationInSeconds := float64(stream.Info.NSamples) / float64(stream.Info.SampleRate) + return int64(durationInSeconds), nil +} + +func GetDuration_MP3(audio io.ReadCloser) (int64, error) { decoder := mp3.NewDecoder(audio) defer func() { if r := recover(); r != nil { @@ -557,3 +653,14 @@ func CalculateAudioDuration(audio io.ReadCloser) (int64, error) { } return int64(totalDuration), nil } + +func GetReadSeeker(audio io.ReadCloser) (io.ReadSeeker, error) { + data, err := ioutil.ReadAll(audio) + if err != nil { + return nil, errors.New("Error in file reader") + } + reader := bytes.NewReader(data) + _, err = reader.Seek(0, io.SeekStart) + + return reader, nil +} diff --git a/go.mod b/go.mod index 7dd25778e3..8bdcbe92bb 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( pgregory.net/rapid v1.1.0 ) -require github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 +require github.com/icza/bitio v1.1.0 // indirect require ( cloud.google.com/go v0.110.2 // indirect @@ -47,6 +47,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/StackExchange/wmi v1.2.1 // indirect + github.com/abema/go-mp4 v1.2.0 github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go v1.44.273 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -72,11 +73,16 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894 // indirect + github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70 github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-audio/audio v1.0.0 // indirect + github.com/go-audio/riff v1.0.0 // indirect + github.com/go-audio/wav v1.1.0 github.com/go-chi/chi/v5 v5.0.12 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -87,6 +93,7 @@ require ( github.com/go-openapi/swag v0.22.8 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/go-test/deep v1.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect @@ -136,6 +143,8 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mewkiz/flac v1.0.10 + github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.66 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -157,6 +166,7 @@ require ( github.com/opencontainers/runc v1.1.5 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c // indirect github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -176,6 +186,7 @@ require ( github.com/status-im/keycard-go v0.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.11 // indirect + github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect @@ -208,4 +219,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1 +replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 diff --git a/go.sum b/go.sum index 40558b8f6c..7f9a145917 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/abema/go-mp4 v1.2.0 h1:gi4X8xg/m179N/J15Fn5ugywN9vtI6PLk6iLldHGLAk= +github.com/abema/go-mp4 v1.2.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -140,11 +142,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -182,8 +186,12 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1 h1:Ao8wONCNgzTpOC79upkZXMf2iSmjFr0Iejb4xHgvZkE= -github.com/eliteprox/ai-worker v0.0.0-20240612192039-abd181a5b6e1/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894 h1:N1Navg94Gvv0DkkFJFoTBxb8e886L3dqq2UoUMjcVZI= +github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894/go.mod h1:nW0Kn5hTb57MDQW6vhOAUsT5/z6o9RQcMs8wmOcZtWw= +github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70 h1:O0HjSbA6P3KVwZsDBn1Kil38PkS9eMOpIhCMyNPk0js= +github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70/go.mod h1:H6o03B1Zd3dem8QXDw0MBAmShfDPkwtzmqUUebZ2HKo= +github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 h1:fNcmANJSD7htUWIhFDpjMQ1Zxr/cbvptcJzmNFfSDeE= +github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -212,6 +220,12 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4= +github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs= +github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA= +github.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498= +github.com/go-audio/wav v1.1.0 h1:jQgLtbqBzY7G+BM8fXF7AHUk1uHUviWS4X39d5rsL2g= +github.com/go-audio/wav v1.1.0/go.mod h1:mpe9qfwbScEbkd8uybLuIpTgHyrISw/OTuvjUW2iGtE= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -253,8 +267,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -376,6 +390,10 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= +github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= +github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= +github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= @@ -477,6 +495,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -504,6 +523,7 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -564,6 +584,10 @@ github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mewkiz/flac v1.0.10 h1:go+Pj8X/HeJm1f9jWhEs484ABhivtjY9s5TYhxWMqNM= +github.com/mewkiz/flac v1.0.10/go.mod h1:l7dt5uFY724eKVkHQtAJAQSkhpC3helU3RDxN0ESAqo= +github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 h1:tnAPMExbRERsyEYkmR1YjhTgDM0iqyiBYf8ojRXxdbA= +github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14/go.mod h1:QYCFBiH5q6XTHEbWhR0uhR3M9qNPoD2CSQzr0g75kE4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= @@ -670,10 +694,14 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw= +github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c h1:AwcgVYzW1T+QuJ2fc55ceOSCiVaOpdYUNpFj9t7+n9U= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= @@ -783,6 +811,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -889,6 +918,7 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjs golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -951,6 +981,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1049,6 +1080,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1058,6 +1090,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1233,6 +1266,8 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/server/ai_http.go b/server/ai_http.go index 3c8e4d97d3..12c387b0b2 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -263,14 +263,12 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request return orch.SpeechToText(ctx, v) } - audio, err := v.Audio.Reader() - if err != nil { - respondWithError(w, err.Error(), http.StatusBadRequest) - } - outPixels, err = common.CalculateAudioDuration(audio) + outPixels, err = common.EstimateAudioDuration(v.Audio) if err != nil { respondWithError(w, "Unable to calculate duration", http.StatusBadRequest) } + outPixels *= 1000 // Convert to milliseconds + default: respondWithError(w, "Unknown request type", http.StatusBadRequest) return diff --git a/server/ai_process.go b/server/ai_process.go index 2d6c1cae00..7b36fac353 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -404,15 +404,11 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes return nil, err } - audio, err := req.Audio.Reader() - if err != nil { - return nil, err - } - - outPixels, err := common.CalculateAudioDuration(audio) + outPixels, err := common.EstimateAudioDuration(req.Audio) if err != nil { return nil, err } + outPixels *= 1000 // Convert to milliseconds setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) if err != nil { return nil, err From 157992042082fb1ef6dd79551d57f2ffcb811c37 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Sun, 30 Jun 2024 15:59:54 -0400 Subject: [PATCH 11/34] Add bad request response for unsupported file types --- common/util.go | 7 ++++--- server/ai_http.go | 1 + server/ai_mediaserver.go | 9 +++++++-- server/ai_process.go | 12 ++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/common/util.go b/common/util.go index 65a6930e6c..8bff15276b 100644 --- a/common/util.go +++ b/common/util.go @@ -540,6 +540,9 @@ func ParseEthAddr(strJsonKey string) (string, error) { } // determines the duration of an mp3 audio file by reading the frames +var ErrUnsupportedFormat = errors.New("Unsupported audio file format. Supported formats: mp3, wav, mp4, m4a, webm, flac") +var ErrorCalculatingDuration = errors.New("Error calculating duration") + func EstimateAudioDuration(audio types.File) (int64, error) { read, err := audio.Reader() if err != nil { @@ -563,12 +566,10 @@ func EstimateAudioDuration(audio types.File) (int64, error) { return GetDuration_MP3(read) default: - return 0, errors.New("unsupported audio format") + return 0, ErrUnsupportedFormat } } -var ErrorCalculatingDuration = errors.New("Error calculating duration") - func GetDuration_WAV(audio io.ReadCloser) (int64, error) { seeker, err := GetReadSeeker(audio) if err != nil { diff --git a/server/ai_http.go b/server/ai_http.go index 12c387b0b2..c350374ab5 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -266,6 +266,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request outPixels, err = common.EstimateAudioDuration(v.Audio) if err != nil { respondWithError(w, "Unable to calculate duration", http.StatusBadRequest) + return } outPixels *= 1000 // Convert to milliseconds diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 895c919fc6..2beeaafcaa 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -352,12 +352,17 @@ func (ls *LivepeerServer) SpeechToText() http.Handler { resp, err := processSpeechToText(ctx, params, req) if err != nil { var e *ServiceUnavailableError + var reqError *BadRequestError if errors.As(err, &e) { respondJsonError(ctx, w, err, http.StatusServiceUnavailable) return + } else if errors.As(err, &reqError) { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } else { + respondJsonError(ctx, w, err, http.StatusInternalServerError) + return } - respondJsonError(ctx, w, err, http.StatusInternalServerError) - return } took := time.Since(start) diff --git a/server/ai_process.go b/server/ai_process.go index 7b36fac353..e54b0a3906 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -34,6 +34,14 @@ type ServiceUnavailableError struct { err error } +type BadRequestError struct { + err error +} + +func (e *BadRequestError) Error() string { + return e.err.Error() +} + func (e *ServiceUnavailableError) Error() string { return e.err.Error() } @@ -546,6 +554,10 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface params.sessManager.Complete(ctx, sess) break } + if errors.Is(err, common.ErrorCalculatingDuration) || errors.Is(err, common.ErrUnsupportedFormat) { + return nil, &BadRequestError{err} + } + clog.Infof(ctx, "Error submitting request cap=%v modelID=%v try=%v orch=%v err=%v", cap, modelID, tries, sess.Transcoder(), err) params.sessManager.Remove(ctx, sess) } From 494654d3443c3ec95892ebe860cedc894f09dcc3 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Mon, 1 Jul 2024 09:28:43 -0400 Subject: [PATCH 12/34] Update name of function --- common/util.go | 2 +- server/ai_http.go | 2 +- server/ai_process.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/util.go b/common/util.go index 8bff15276b..0218ebea73 100644 --- a/common/util.go +++ b/common/util.go @@ -543,7 +543,7 @@ func ParseEthAddr(strJsonKey string) (string, error) { var ErrUnsupportedFormat = errors.New("Unsupported audio file format. Supported formats: mp3, wav, mp4, m4a, webm, flac") var ErrorCalculatingDuration = errors.New("Error calculating duration") -func EstimateAudioDuration(audio types.File) (int64, error) { +func CalculateAudioDuration(audio types.File) (int64, error) { read, err := audio.Reader() if err != nil { return 0, err diff --git a/server/ai_http.go b/server/ai_http.go index c350374ab5..7bf049f0fe 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -263,7 +263,7 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request return orch.SpeechToText(ctx, v) } - outPixels, err = common.EstimateAudioDuration(v.Audio) + outPixels, err = common.CalculateAudioDuration(v.Audio) if err != nil { respondWithError(w, "Unable to calculate duration", http.StatusBadRequest) return diff --git a/server/ai_process.go b/server/ai_process.go index e54b0a3906..2f9f10a731 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -412,7 +412,7 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes return nil, err } - outPixels, err := common.EstimateAudioDuration(req.Audio) + outPixels, err := common.CalculateAudioDuration(req.Audio) if err != nil { return nil, err } From 63c20e373f52d0c04b65559d199bdd2d81859ff6 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Tue, 2 Jul 2024 19:24:37 +0000 Subject: [PATCH 13/34] Update go mod to ai-runner --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8bdcbe92bb..443b7817f7 100644 --- a/go.mod +++ b/go.mod @@ -219,4 +219,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 +replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240702184631-61c475f5b608 diff --git a/go.sum b/go.sum index 7f9a145917..1ec976ae26 100644 --- a/go.sum +++ b/go.sum @@ -190,8 +190,8 @@ github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894 h1:N1Navg94Gvv0DkkFJF github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894/go.mod h1:nW0Kn5hTb57MDQW6vhOAUsT5/z6o9RQcMs8wmOcZtWw= github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70 h1:O0HjSbA6P3KVwZsDBn1Kil38PkS9eMOpIhCMyNPk0js= github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70/go.mod h1:H6o03B1Zd3dem8QXDw0MBAmShfDPkwtzmqUUebZ2HKo= -github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 h1:fNcmANJSD7htUWIhFDpjMQ1Zxr/cbvptcJzmNFfSDeE= -github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/eliteprox/ai-worker v0.0.0-20240702184631-61c475f5b608 h1:ZPWBCV/mUac6QqM6hwWNKxMuuKNmlTlEjs4bBxnmRMY= +github.com/eliteprox/ai-worker v0.0.0-20240702184631-61c475f5b608/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From b78f11f1da3ccfb64da9e809716eb0481cfe2090 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 3 Jul 2024 04:11:04 -0400 Subject: [PATCH 14/34] Use ffmpeg to get duration --- common/util.go | 124 ++++--------------------------------------------- go.mod | 18 ++----- go.sum | 49 +++---------------- 3 files changed, 19 insertions(+), 172 deletions(-) diff --git a/common/util.go b/common/util.go index 0218ebea73..962358c995 100644 --- a/common/util.go +++ b/common/util.go @@ -1,7 +1,6 @@ package common import ( - "bytes" "context" "encoding/hex" "encoding/json" @@ -12,7 +11,6 @@ import ( "math/big" "math/rand" "mime" - "path/filepath" "regexp" "sort" "strconv" @@ -20,20 +18,15 @@ import ( "testing" "time" - "github.com/abema/go-mp4" - "github.com/ebml-go/webm" "github.com/ethereum/go-ethereum/crypto" - "github.com/go-audio/wav" "github.com/golang/glog" "github.com/jaypipes/ghw" "github.com/jaypipes/ghw/pkg/gpu" "github.com/jaypipes/ghw/pkg/pci" "github.com/livepeer/go-livepeer/net" ffmpeg "github.com/livepeer/lpms/ffmpeg" - "github.com/mewkiz/flac" "github.com/oapi-codegen/runtime/types" "github.com/pkg/errors" - "github.com/tcolgate/mp3" "google.golang.org/grpc/peer" ) @@ -550,118 +543,19 @@ func CalculateAudioDuration(audio types.File) (int64, error) { } defer read.Close() - fileExt := filepath.Ext(audio.Filename()) - switch fileExt { - case ".wav": - return GetDuration_WAV(read) - case ".mp4": - return GetDuration_MP4(read) - case ".m4a": - return GetDuration_MP4(read) - case ".webm": - return GetDuration_WEBM(read) - case ".flac": - return GetDuration_FLAC(read) - case ".mp3": - return GetDuration_MP3(read) - - default: - return 0, ErrUnsupportedFormat - } -} - -func GetDuration_WAV(audio io.ReadCloser) (int64, error) { - seeker, err := GetReadSeeker(audio) - if err != nil { - return 0, ErrorCalculatingDuration - } - - decoder := wav.NewDecoder(seeker) - decoder.ReadInfo() - duration, err := decoder.Duration() + bytearr, _ := audio.Bytes() + _, mediaFormat, err := ffmpeg.GetCodecInfoBytes(bytearr) if err != nil { - return 0, ErrorCalculatingDuration + return 0, errors.New("Error getting codec info url") } - return int64(duration.Seconds()), nil -} - -func GetDuration_MP4(audio io.ReadCloser) (int64, error) { - seeker, err := GetReadSeeker(audio) - if err != nil { - return 0, ErrorCalculatingDuration - } - - path := mp4.BoxPath{mp4.BoxTypeMoov(), mp4.BoxTypeMvhd()} - boxes, err := mp4.ExtractBoxWithPayload(seeker, nil, path) - for _, box := range boxes { - if box.Info.Type == mp4.BoxTypeMvhd() { - if mvhdBox, ok := box.Payload.(*mp4.Mvhd); ok { - durationInSeconds := float64(mvhdBox.GetDuration()) / float64(mvhdBox.Timescale) - return int64(durationInSeconds), nil - } - } - } - - return 0, ErrorCalculatingDuration -} - -func GetDuration_WEBM(audio io.ReadCloser) (int64, error) { - seeker, err := GetReadSeeker(audio) - if err != nil { - return 0, ErrorCalculatingDuration - } - - var webmData webm.WebM - if _, err := webm.Parse(seeker, &webmData); err != nil { - return 0, ErrorCalculatingDuration - } - - return int64(webmData.GetDuration().Seconds()), nil -} - -func GetDuration_FLAC(audio io.ReadCloser) (int64, error) { - stream, err := flac.Parse(audio) - if err != nil { - return 0, ErrorCalculatingDuration - } - - durationInSeconds := float64(stream.Info.NSamples) / float64(stream.Info.SampleRate) - return int64(durationInSeconds), nil -} - -func GetDuration_MP3(audio io.ReadCloser) (int64, error) { - decoder := mp3.NewDecoder(audio) - defer func() { - if r := recover(); r != nil { - audio.Close() - return - } - }() - totalDuration := 0.0 - var frame mp3.Frame - var skipped int - var frameCount int - for { - if err := decoder.Decode(&frame, &skipped); err != nil { - if err == io.EOF { - break - } - return 0, err - } - totalDuration += frame.Duration().Seconds() - frameCount++ + //TODO: Develop some input validation logic + if len(mediaFormat.Vcodec) == 0 { + return 0, ErrUnsupportedFormat } - return int64(totalDuration), nil -} -func GetReadSeeker(audio io.ReadCloser) (io.ReadSeeker, error) { - data, err := ioutil.ReadAll(audio) - if err != nil { - return nil, errors.New("Error in file reader") - } - reader := bytes.NewReader(data) - _, err = reader.Seek(0, io.SeekStart) + // log the duration + glog.Infof("Audio duration: %d", int64(mediaFormat.Dur)) - return reader, nil + return int64(mediaFormat.Dur), nil } diff --git a/go.mod b/go.mod index 8bdcbe92bb..f75063235b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/getkin/kin-openapi v0.124.0 github.com/golang/glog v1.1.1 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 github.com/jaypipes/ghw v0.10.0 github.com/jaypipes/pcidb v1.0.0 github.com/livepeer/ai-worker v0.0.8 @@ -32,12 +32,10 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/net v0.25.0 google.golang.org/grpc v1.57.1 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.33.0 pgregory.net/rapid v1.1.0 ) -require github.com/icza/bitio v1.1.0 // indirect - require ( cloud.google.com/go v0.110.2 // indirect cloud.google.com/go/compute v1.20.0 // indirect @@ -47,7 +45,6 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/StackExchange/wmi v1.2.1 // indirect - github.com/abema/go-mp4 v1.2.0 github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go v1.44.273 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -73,16 +70,11 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894 // indirect - github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70 github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-audio/audio v1.0.0 // indirect - github.com/go-audio/riff v1.0.0 // indirect - github.com/go-audio/wav v1.1.0 github.com/go-chi/chi/v5 v5.0.12 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -143,8 +135,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/mewkiz/flac v1.0.10 - github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.66 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -166,7 +156,6 @@ require ( github.com/opencontainers/runc v1.1.5 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect - github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c // indirect github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -186,7 +175,6 @@ require ( github.com/status-im/keycard-go v0.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/supranational/blst v0.3.11 // indirect - github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect @@ -220,3 +208,5 @@ require ( ) replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 + +replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184 diff --git a/go.sum b/go.sum index 7f9a145917..583099876e 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,6 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= -github.com/abema/go-mp4 v1.2.0 h1:gi4X8xg/m179N/J15Fn5ugywN9vtI6PLk6iLldHGLAk= -github.com/abema/go-mp4 v1.2.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -142,13 +140,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -186,12 +182,10 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894 h1:N1Navg94Gvv0DkkFJFoTBxb8e886L3dqq2UoUMjcVZI= -github.com/ebml-go/ebml v0.0.0-20160925193348-ca8851a10894/go.mod h1:nW0Kn5hTb57MDQW6vhOAUsT5/z6o9RQcMs8wmOcZtWw= -github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70 h1:O0HjSbA6P3KVwZsDBn1Kil38PkS9eMOpIhCMyNPk0js= -github.com/ebml-go/webm v0.0.0-20221117133942-84fa5245cf70/go.mod h1:H6o03B1Zd3dem8QXDw0MBAmShfDPkwtzmqUUebZ2HKo= github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 h1:fNcmANJSD7htUWIhFDpjMQ1Zxr/cbvptcJzmNFfSDeE= github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184 h1:Zv6GVbT/nOo6KoGAaKlndQK7nhedCIQFqoTwVlbESG0= +github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -220,12 +214,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4= -github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs= -github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA= -github.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498= -github.com/go-audio/wav v1.1.0 h1:jQgLtbqBzY7G+BM8fXF7AHUk1uHUviWS4X39d5rsL2g= -github.com/go-audio/wav v1.1.0/go.mod h1:mpe9qfwbScEbkd8uybLuIpTgHyrISw/OTuvjUW2iGtE= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -313,8 +301,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -390,10 +378,6 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= -github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= -github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= -github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= @@ -495,7 +479,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -523,7 +506,6 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -559,8 +541,6 @@ github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cO github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded/go.mod h1:xkDdm+akniYxVT9KW1Y2Y7Hso6aW+rZObz3nrA9yTHw= github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 h1:4oH3NqV0NvcdS44Ld3zK2tO8IUiNozIggm74yobQeZg= github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18/go.mod h1:Jpf4jHK+fbWioBHRDRM1WadNT1qmY27g2YicTdO0Rtc= -github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 h1:4A6geMb+HfxBBfaS24t8R3ddpEDfWbpx7NTQZMt5Fp4= -github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/livepeer/m3u8 v0.11.1 h1:VkUJzfNTyjy9mqsgp5JPvouwna8wGZMvd/gAfT5FinU= github.com/livepeer/m3u8 v0.11.1/go.mod h1:IUqAtwWPAG2CblfQa4SVzTQoDcEMPyfNOaBSxqHMS04= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -584,10 +564,6 @@ github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mewkiz/flac v1.0.10 h1:go+Pj8X/HeJm1f9jWhEs484ABhivtjY9s5TYhxWMqNM= -github.com/mewkiz/flac v1.0.10/go.mod h1:l7dt5uFY724eKVkHQtAJAQSkhpC3helU3RDxN0ESAqo= -github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 h1:tnAPMExbRERsyEYkmR1YjhTgDM0iqyiBYf8ojRXxdbA= -github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14/go.mod h1:QYCFBiH5q6XTHEbWhR0uhR3M9qNPoD2CSQzr0g75kE4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= @@ -694,14 +670,10 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw= -github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= -github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c h1:AwcgVYzW1T+QuJ2fc55ceOSCiVaOpdYUNpFj9t7+n9U= -github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= @@ -811,14 +783,11 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= -github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300 h1:XQdibLKagjdevRB6vAjVY4qbSr8rQ610YzTkWcxzxSI= -github.com/tcolgate/mp3 v0.0.0-20170426193717-e79c5a46d300/go.mod h1:FNa/dfN95vAYCNFrIKRrlRo+MBLbwmR9Asa5f2ljmBI= github.com/testcontainers/testcontainers-go v0.9.0 h1:ZyftCfROjGrKlxk3MOUn2DAzWrUtzY/mj17iAkdUIvI= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -918,7 +887,6 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjs golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -981,7 +949,6 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1080,7 +1047,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1090,7 +1056,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1252,8 +1217,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1266,8 +1231,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 3fea27b90ca3103b987d18e750bad31e1418960b Mon Sep 17 00:00:00 2001 From: 0xb79orch <0xb79orch@gmail.com> Date: Wed, 3 Jul 2024 19:17:27 +0000 Subject: [PATCH 15/34] update install_ffmpeg.sh to parse audio better --- install_ffmpeg.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_ffmpeg.sh b/install_ffmpeg.sh index bb99292ef0..b33438d8ec 100755 --- a/install_ffmpeg.sh +++ b/install_ffmpeg.sh @@ -208,13 +208,13 @@ if [[ ! -e "$ROOT/ffmpeg/libavcodec/libavcodec.a" ]]; then ./configure ${TARGET_OS:-} $DISABLE_FFMPEG_COMPONENTS --fatal-warnings \ --enable-libx264 --enable-gpl \ --enable-protocol=rtmp,file,pipe \ - --enable-muxer=mpegts,hls,segment,mp4,hevc,matroska,webm,null --enable-demuxer=flv,mpegts,mp4,mov,webm,matroska,image2 \ + --enable-muxer=mp3,wav,flac,mpegts,hls,segment,mp4,hevc,matroska,webm,null --enable-demuxer=mp3,wav,flac,flv,mpegts,mp4,mov,webm,matroska,image2 \ --enable-bsf=h264_mp4toannexb,aac_adtstoasc,h264_metadata,h264_redundant_pps,hevc_mp4toannexb,extract_extradata \ - --enable-parser=aac,aac_latm,h264,hevc,vp8,vp9,png \ + --enable-parser=mpegaudio,aac,aac_latm,h264,hevc,vp8,vp9,png \ --enable-filter=abuffer,buffer,abuffersink,buffersink,afifo,fifo,aformat,format \ --enable-filter=aresample,asetnsamples,fps,scale,hwdownload,select,livepeer_dnn,signature \ --enable-encoder=aac,opus,libx264 \ - --enable-decoder=aac,opus,h264,png \ + --enable-decoder=mp3,vorbis,aac,opus,h264,png \ --extra-cflags="${EXTRA_CFLAGS} -I${ROOT}/compiled/include -I/usr/local/cuda/include" \ --extra-ldflags="${EXTRA_FFMPEG_LDFLAGS} -L${ROOT}/compiled/lib -L/usr/local/cuda/lib64" \ --prefix="$ROOT/compiled" \ From c00b210d6ebd59ff6fc43adadcde1694a6d729d7 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 3 Jul 2024 19:52:35 -0400 Subject: [PATCH 16/34] Check for audio codec instead of video codec --- common/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/util.go b/common/util.go index 962358c995..7217ff2100 100644 --- a/common/util.go +++ b/common/util.go @@ -550,7 +550,7 @@ func CalculateAudioDuration(audio types.File) (int64, error) { } //TODO: Develop some input validation logic - if len(mediaFormat.Vcodec) == 0 { + if len(mediaFormat.Acodec) == 0 { return 0, ErrUnsupportedFormat } From 29309eb429b93315049c31179650d734e04fe94f Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 3 Jul 2024 19:53:18 -0400 Subject: [PATCH 17/34] gomod edits --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f75063235b..77ec52fd20 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 -replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184 +replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 diff --git a/go.sum b/go.sum index 583099876e..a94bd3bfa1 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 h1:fNcmANJSD7htUWIhFDpjMQ1Zxr/cbvptcJzmNFfSDeE= github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184 h1:Zv6GVbT/nOo6KoGAaKlndQK7nhedCIQFqoTwVlbESG0= -github.com/eliteprox/lpms v0.0.0-20240703072932-1fcc47e1e184/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= +github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 h1:IBmvTlg33ZL0OEjT+BCn+tVQLUSZ2p+li1Sq9rr8eiI= +github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 9920ccadb32ea3c8e09f15b2c21dfbeb25e20270 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 3 Jul 2024 21:39:57 -0400 Subject: [PATCH 18/34] add docker file --- docker/Dockerfile.local.ai-worker | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 docker/Dockerfile.local.ai-worker diff --git a/docker/Dockerfile.local.ai-worker b/docker/Dockerfile.local.ai-worker new file mode 100644 index 0000000000..922f20e8ca --- /dev/null +++ b/docker/Dockerfile.local.ai-worker @@ -0,0 +1,71 @@ +FROM --platform=$BUILDPLATFORM livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 as build + +ARG TARGETARCH +ARG BUILDARCH + +ENV GOARCH="$TARGETARCH" \ + PATH="/usr/local/go/bin:/go/bin:${PATH}" \ + PKG_CONFIG_PATH="/root/compiled/lib/pkgconfig" \ + CPATH="/usr/local/cuda_${TARGETARCH}/include" \ + LIBRARY_PATH="/usr/local/cuda_${TARGETARCH}/lib64" \ + DEBIAN_FRONTEND="noninteractive" \ + CGO_LDFLAGS="-L/usr/local/cuda_${TARGETARCH}/lib64" + +RUN apt update \ + && apt install -yqq software-properties-common curl apt-transport-https lsb-release yasm \ + && curl -fsSL https://dl.google.com/go/go1.21.5.linux-${BUILDARCH}.tar.gz | tar -C /usr/local -xz \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb [arch=${BUILDARCH}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ + && add-apt-repository "deb [arch=${BUILDARCH}] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \ + && apt update \ + && apt -yqq install clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python docker-ce-cli pciutils gcc-multilib libgcc-8-dev-arm64-cross gcc-mingw-w64-x86-64 zlib1g zlib1g-dev + +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \ + && update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30 + +RUN GRPC_HEALTH_PROBE_VERSION=v0.3.6 \ + && curl -fsSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} -o /usr/bin/grpc_health_probe \ + && chmod +x /usr/bin/grpc_health_probe \ + && ldconfig /usr/local/lib + +ENV GOPATH=/go \ + GO_BUILD_DIR=/build/ \ + GOFLAGS="-mod=readonly" + +WORKDIR /src +COPY ./ai-worker ./ai-worker +RUN mkdir -p /go \ + && curl -fsSLO https://github.com/livepeer/livepeer-ml/releases/download/v0.3/tasmodel.pb + +COPY go-livepeer/install_ffmpeg.sh ./install_ffmpeg.sh + +ARG BUILD_TAGS +ENV BUILD_TAGS=${BUILD_TAGS} + +COPY ./go-livepeer/go.mod ./go-livepeer/go.sum ./ +RUN go mod download + +RUN ./install_ffmpeg.sh \ + && GO111MODULE=on go get -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 \ + && go get -v github.com/jstemmer/go-junit-report + +COPY ./go-livepeer . + +RUN make livepeer livepeer_cli livepeer_bench livepeer_router + +FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-amd64-base + +FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-arm64-base + +FROM livepeer-${TARGETARCH}-base + +ENV NVIDIA_DRIVER_CAPABILITIES=all + +COPY --from=build /build/ /usr/local/bin/ +COPY --from=build /usr/bin/grpc_health_probe /usr/local/bin/grpc_health_probe +COPY --from=build /src/tasmodel.pb /tasmodel.pb +COPY --from=build /usr/share/misc/pci.ids /usr/share/misc/pci.ids + +ENTRYPOINT ["/usr/local/bin/livepeer"] From a185b5d2fcd450e70b0067c90fce2a4dfc67e9a5 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Thu, 4 Jul 2024 17:46:29 -0400 Subject: [PATCH 19/34] Update install_ffmpeg.sh to improve audio support, Add duration validation and logging, pin lpms --- common/util.go | 15 ++++++--------- go.mod | 2 +- go.sum | 4 ++-- install_ffmpeg.sh | 6 +++--- server/ai_process.go | 2 ++ 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/common/util.go b/common/util.go index 7217ff2100..74d2d9def2 100644 --- a/common/util.go +++ b/common/util.go @@ -533,7 +533,7 @@ func ParseEthAddr(strJsonKey string) (string, error) { } // determines the duration of an mp3 audio file by reading the frames -var ErrUnsupportedFormat = errors.New("Unsupported audio file format. Supported formats: mp3, wav, mp4, m4a, webm, flac") +var ErrUnsupportedFormat = errors.New("Unsupported audio file format") var ErrorCalculatingDuration = errors.New("Error calculating duration") func CalculateAudioDuration(audio types.File) (int64, error) { @@ -546,16 +546,13 @@ func CalculateAudioDuration(audio types.File) (int64, error) { bytearr, _ := audio.Bytes() _, mediaFormat, err := ffmpeg.GetCodecInfoBytes(bytearr) if err != nil { - return 0, errors.New("Error getting codec info url") + return 0, errors.New("Error getting codec info") } - //TODO: Develop some input validation logic - if len(mediaFormat.Acodec) == 0 { - return 0, ErrUnsupportedFormat + duration := int64(mediaFormat.Dur) + if duration <= 0 { + return 0, ErrorCalculatingDuration } - // log the duration - glog.Infof("Audio duration: %d", int64(mediaFormat.Dur)) - - return int64(mediaFormat.Dur), nil + return duration, nil } diff --git a/go.mod b/go.mod index 77ec52fd20..651bb57bfe 100644 --- a/go.mod +++ b/go.mod @@ -207,6 +207,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/livepeer/ai-worker v0.0.8 => github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 +replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 diff --git a/go.sum b/go.sum index a94bd3bfa1..285f508567 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60 h1:fNcmANJSD7htUWIhFDpjMQ1Zxr/cbvptcJzmNFfSDeE= -github.com/eliteprox/ai-worker v0.0.0-20240627194509-7748c9f4fa60/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a h1:zzbedC+YkkcZ/odcPoW6xikDUxUzEHFUHPoidxDAbL4= +github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 h1:IBmvTlg33ZL0OEjT+BCn+tVQLUSZ2p+li1Sq9rr8eiI= github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/install_ffmpeg.sh b/install_ffmpeg.sh index b33438d8ec..df864b4968 100755 --- a/install_ffmpeg.sh +++ b/install_ffmpeg.sh @@ -210,11 +210,11 @@ if [[ ! -e "$ROOT/ffmpeg/libavcodec/libavcodec.a" ]]; then --enable-protocol=rtmp,file,pipe \ --enable-muxer=mp3,wav,flac,mpegts,hls,segment,mp4,hevc,matroska,webm,null --enable-demuxer=mp3,wav,flac,flv,mpegts,mp4,mov,webm,matroska,image2 \ --enable-bsf=h264_mp4toannexb,aac_adtstoasc,h264_metadata,h264_redundant_pps,hevc_mp4toannexb,extract_extradata \ - --enable-parser=mpegaudio,aac,aac_latm,h264,hevc,vp8,vp9,png \ + --enable-parser=mpegaudio,vorbis,opus,flac,aac,aac_latm,h264,hevc,vp8,vp9,png \ --enable-filter=abuffer,buffer,abuffersink,buffersink,afifo,fifo,aformat,format \ --enable-filter=aresample,asetnsamples,fps,scale,hwdownload,select,livepeer_dnn,signature \ - --enable-encoder=aac,opus,libx264 \ - --enable-decoder=mp3,vorbis,aac,opus,h264,png \ + --enable-encoder=mp3,vorbis,flac,aac,opus,libx264 \ + --enable-decoder=mp3,vorbis,flac,aac,opus,h264,png \ --extra-cflags="${EXTRA_CFLAGS} -I${ROOT}/compiled/include -I/usr/local/cuda/include" \ --extra-ldflags="${EXTRA_FFMPEG_LDFLAGS} -L${ROOT}/compiled/lib -L/usr/local/cuda/lib64" \ --prefix="$ROOT/compiled" \ diff --git a/server/ai_process.go b/server/ai_process.go index 2f9f10a731..88443d3938 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/golang/glog" "github.com/livepeer/ai-worker/worker" "github.com/livepeer/go-livepeer/clog" "github.com/livepeer/go-livepeer/common" @@ -416,6 +417,7 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes if err != nil { return nil, err } + glog.Infof("Submitting speech-to-text media with duration: %d seconds", outPixels) outPixels *= 1000 // Convert to milliseconds setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) if err != nil { From 7f1882098e2857a187803241fa6c64c94c774891 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Thu, 4 Jul 2024 19:45:24 -0400 Subject: [PATCH 20/34] rename speech-to-text to audio-to-text --- cmd/livepeer/starter/starter.go | 12 ++++++------ core/ai.go | 2 +- core/capabilities.go | 6 +++--- core/orchestrator.go | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- server/ai_http.go | 14 +++++++------- server/ai_mediaserver.go | 12 ++++++------ server/ai_process.go | 20 ++++++++++---------- server/rpc.go | 2 +- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 34e09fa232..945d0dee58 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -613,18 +613,18 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { constraints[core.Capability_Upscale].Models[config.ModelID] = modelConstraint n.SetBasePriceForCap("default", core.Capability_Upscale, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) - case "speech-to-text": - _, ok := constraints[core.Capability_SpeechToText] + case "audio-to-text": + _, ok := constraints[core.Capability_AudioToText] if !ok { - aiCaps = append(aiCaps, core.Capability_SpeechToText) - constraints[core.Capability_SpeechToText] = &core.Constraints{ + aiCaps = append(aiCaps, core.Capability_AudioToText) + constraints[core.Capability_AudioToText] = &core.Constraints{ Models: make(map[string]*core.ModelConstraint), } } - constraints[core.Capability_SpeechToText].Models[config.ModelID] = modelConstraint + constraints[core.Capability_AudioToText].Models[config.ModelID] = modelConstraint - n.SetBasePriceForCap("default", core.Capability_SpeechToText, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) + n.SetBasePriceForCap("default", core.Capability_AudioToText, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) } diff --git a/core/ai.go b/core/ai.go index 9a8807ff44..fde3060a96 100644 --- a/core/ai.go +++ b/core/ai.go @@ -16,7 +16,7 @@ type AI interface { ImageToImage(context.Context, worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) ImageToVideo(context.Context, worker.ImageToVideoMultipartRequestBody) (*worker.VideoResponse, error) Upscale(context.Context, worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error) - SpeechToText(context.Context, worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) + AudioToText(context.Context, worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) Warm(context.Context, string, string, worker.RunnerEndpoint, worker.OptimizationFlags) error Stop(context.Context) error HasCapacity(pipeline, modelID string) bool diff --git a/core/capabilities.go b/core/capabilities.go index 3615a30711..12eb7bbc6d 100644 --- a/core/capabilities.go +++ b/core/capabilities.go @@ -71,7 +71,7 @@ const ( Capability_ImageToImage Capability_ImageToVideo Capability_Upscale - Capability_SpeechToText + Capability_AudioToText ) var CapabilityNameLookup = map[Capability]string{ @@ -107,7 +107,7 @@ var CapabilityNameLookup = map[Capability]string{ Capability_ImageToImage: "Image to image", Capability_ImageToVideo: "Image to video", Capability_Upscale: "Upscale", - Capability_SpeechToText: "Speech to text", + Capability_AudioToText: "Audio to text", } var CapabilityTestLookup = map[Capability]CapabilityTest{ @@ -197,7 +197,7 @@ func OptionalCapabilities() []Capability { Capability_ImageToImage, Capability_ImageToVideo, Capability_Upscale, - Capability_SpeechToText, + Capability_AudioToText, } } diff --git a/core/orchestrator.go b/core/orchestrator.go index 6086067b70..9505e97b30 100644 --- a/core/orchestrator.go +++ b/core/orchestrator.go @@ -126,8 +126,8 @@ func (orch *orchestrator) Upscale(ctx context.Context, req worker.UpscaleMultipa return orch.node.upscale(ctx, req) } -func (orch *orchestrator) SpeechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { - return orch.node.speechToText(ctx, req) +func (orch *orchestrator) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { + return orch.node.AudioToText(ctx, req) } func (orch *orchestrator) ProcessPayment(ctx context.Context, payment net.Payment, manifestID ManifestID) error { @@ -949,9 +949,9 @@ func (n *LivepeerNode) upscale(ctx context.Context, req worker.UpscaleMultipartR return n.AIWorker.Upscale(ctx, req) } -func (n *LivepeerNode) speechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { +func (n *LivepeerNode) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { - resp, err := n.AIWorker.SpeechToText(ctx, req) + resp, err := n.AIWorker.AudioToText(ctx, req) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 651bb57bfe..eba0a0c377 100644 --- a/go.mod +++ b/go.mod @@ -207,6 +207,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a +replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 diff --git a/go.sum b/go.sum index 285f508567..3657a1bc81 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a h1:zzbedC+YkkcZ/odcPoW6xikDUxUzEHFUHPoidxDAbL4= -github.com/eliteprox/ai-worker v0.0.0-20240704214018-ce7332b2492a/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 h1:q0D+mEgnwRdK8MxTnB2qQ15YhSazp0EFN2BH6eud+ig= +github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 h1:IBmvTlg33ZL0OEjT+BCn+tVQLUSZ2p+li1Sq9rr8eiI= github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/server/ai_http.go b/server/ai_http.go index 7bf049f0fe..b23bc61794 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -42,7 +42,7 @@ func startAIServer(lp lphttp) error { lp.transRPC.Handle("/image-to-image", oapiReqValidator(lp.ImageToImage())) lp.transRPC.Handle("/image-to-video", oapiReqValidator(lp.ImageToVideo())) lp.transRPC.Handle("/upscale", oapiReqValidator(lp.Upscale())) - lp.transRPC.Handle("/speech-to-text", oapiReqValidator(lp.SpeechToText())) + lp.transRPC.Handle("/audio-to-text", oapiReqValidator(lp.AudioToText())) return nil } @@ -133,7 +133,7 @@ func (h *lphttp) Upscale() http.Handler { }) } -func (h *lphttp) SpeechToText() http.Handler { +func (h *lphttp) AudioToText() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { orch := h.orchestrator @@ -146,7 +146,7 @@ func (h *lphttp) SpeechToText() http.Handler { return } - var req worker.SpeechToTextMultipartRequestBody + var req worker.AudioToTextMultipartRequestBody if err := runtime.BindMultipart(&req, *multiRdr); err != nil { respondWithError(w, err.Error(), http.StatusInternalServerError) return @@ -255,12 +255,12 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request frames := int64(25) outPixels = height * width * int64(frames) - case worker.SpeechToTextMultipartRequestBody: - pipeline = "speech-to-text" - cap = core.Capability_SpeechToText + case worker.AudioToTextMultipartRequestBody: + pipeline = "audio-to-text" + cap = core.Capability_AudioToText modelID = *v.ModelId submitFn = func(ctx context.Context) (interface{}, error) { - return orch.SpeechToText(ctx, v) + return orch.AudioToText(ctx, v) } outPixels, err = common.CalculateAudioDuration(v.Audio) diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 2beeaafcaa..3b068d48f1 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -68,7 +68,7 @@ func startAIMediaServer(ls *LivepeerServer) error { ls.HTTPMux.Handle("/upscale", oapiReqValidator(ls.Upscale())) ls.HTTPMux.Handle("/image-to-video", oapiReqValidator(ls.ImageToVideo())) ls.HTTPMux.Handle("/image-to-video/result", ls.ImageToVideoResult()) - ls.HTTPMux.Handle("/speech-to-text", oapiReqValidator(ls.SpeechToText())) + ls.HTTPMux.Handle("/audio-to-text", oapiReqValidator(ls.AudioToText())) return nil } @@ -321,7 +321,7 @@ func (ls *LivepeerServer) Upscale() http.Handler { }) } -func (ls *LivepeerServer) SpeechToText() http.Handler { +func (ls *LivepeerServer) AudioToText() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) @@ -334,13 +334,13 @@ func (ls *LivepeerServer) SpeechToText() http.Handler { return } - var req worker.SpeechToTextMultipartRequestBody + var req worker.AudioToTextMultipartRequestBody if err := runtime.BindMultipart(&req, *multiRdr); err != nil { respondJsonError(ctx, w, err, http.StatusBadRequest) return } - clog.V(common.VERBOSE).Infof(ctx, "Received SpeechToText request model_id=%v", *req.ModelId) + clog.V(common.VERBOSE).Infof(ctx, "Received AudioToText request model_id=%v", *req.ModelId) params := aiRequestParams{ node: ls.LivepeerNode, @@ -349,7 +349,7 @@ func (ls *LivepeerServer) SpeechToText() http.Handler { } start := time.Now() - resp, err := processSpeechToText(ctx, params, req) + resp, err := processAudioToText(ctx, params, req) if err != nil { var e *ServiceUnavailableError var reqError *BadRequestError @@ -366,7 +366,7 @@ func (ls *LivepeerServer) SpeechToText() http.Handler { } took := time.Since(start) - clog.V(common.VERBOSE).Infof(ctx, "Processed SpeechToText request model_id=%v took=%v", *req.ModelId, took) + clog.V(common.VERBOSE).Infof(ctx, "Processed AudioToText request model_id=%v took=%v", *req.ModelId, took) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/server/ai_process.go b/server/ai_process.go index 88443d3938..ae6689b752 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -29,7 +29,7 @@ const defaultTextToImageModelID = "stabilityai/sdxl-turbo" const defaultImageToImageModelID = "stabilityai/sdxl-turbo" const defaultImageToVideoModelID = "stabilityai/stable-video-diffusion-img2vid-xt" const defaultUpscaleModelID = "stabilityai/stable-diffusion-x4-upscaler" -const defaultSpeechToTextModelID = "openai/whisper-large-v3" +const defaultAudioToTextModelID = "openai/whisper-large-v3" type ServiceUnavailableError struct { err error @@ -401,9 +401,9 @@ func submitUpscale(ctx context.Context, params aiRequestParams, sess *AISession, return resp.JSON200, nil } -func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISession, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { +func submitAudioToText(ctx context.Context, params aiRequestParams, sess *AISession, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { var buf bytes.Buffer - mw, err := worker.NewSpeechToTextMultipartWriter(&buf, req) + mw, err := worker.NewAudioToTextMultipartWriter(&buf, req) if err != nil { return nil, err } @@ -417,7 +417,7 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes if err != nil { return nil, err } - glog.Infof("Submitting speech-to-text media with duration: %d seconds", outPixels) + glog.Infof("Submitting audio-to-text media with duration: %d seconds", outPixels) outPixels *= 1000 // Convert to milliseconds setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) if err != nil { @@ -426,7 +426,7 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes defer completeBalanceUpdate(sess.BroadcastSession, balUpdate) start := time.Now() - resp, err := client.SpeechToTextWithBody(ctx, mw.FormDataContentType(), &buf, setHeaders) + resp, err := client.AudioToTextWithBody(ctx, mw.FormDataContentType(), &buf, setHeaders) took := time.Since(start) if err != nil { return nil, err @@ -458,7 +458,7 @@ func submitSpeechToText(ctx context.Context, params aiRequestParams, sess *AISes return &res, nil } -func processSpeechToText(ctx context.Context, params aiRequestParams, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) { +func processAudioToText(ctx context.Context, params aiRequestParams, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { resp, err := processAIRequest(ctx, params, req) if err != nil { return nil, err @@ -512,14 +512,14 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitUpscale(ctx, params, sess, v) } - case worker.SpeechToTextMultipartRequestBody: - cap = core.Capability_SpeechToText - modelID = defaultSpeechToTextModelID + case worker.AudioToTextMultipartRequestBody: + cap = core.Capability_AudioToText + modelID = defaultAudioToTextModelID if v.ModelId != nil { modelID = *v.ModelId } submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { - return submitSpeechToText(ctx, params, sess, v) + return submitAudioToText(ctx, params, sess, v) } // Add more cases as needed... default: diff --git a/server/rpc.go b/server/rpc.go index ff5b21ac6d..0fc46494f8 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -67,7 +67,7 @@ type Orchestrator interface { ImageToImage(ctx context.Context, req worker.ImageToImageMultipartRequestBody) (*worker.ImageResponse, error) ImageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) Upscale(ctx context.Context, req worker.UpscaleMultipartRequestBody) (*worker.ImageResponse, error) - SpeechToText(ctx context.Context, req worker.SpeechToTextMultipartRequestBody) (*worker.TextResponse, error) + AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) } // Balance describes methods for a session's balance maintenance From 1a6c5008d15d3df705e009edb650d80d5f1d2e55 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Fri, 5 Jul 2024 00:14:09 -0400 Subject: [PATCH 21/34] Update go-mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eba0a0c377..9a3e93102a 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 -replace github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 => github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 +replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e diff --git a/go.sum b/go.sum index 3657a1bc81..60b4e35422 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 h1:q0D+mEgnwRdK8MxTnB2qQ15YhSazp0EFN2BH6eud+ig= github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3 h1:IBmvTlg33ZL0OEjT+BCn+tVQLUSZ2p+li1Sq9rr8eiI= -github.com/eliteprox/lpms v0.0.0-20240703230710-0e84f60a46a3/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= +github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e h1:KeX9oyrfxV63TQQPpr7dARppyzNkAcepwYScmfRh7Ns= +github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 6d534bcd910a3284fb933a6eb4b2057a1e4a864e Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Fri, 5 Jul 2024 00:19:54 -0400 Subject: [PATCH 22/34] cleanup --- docker/Dockerfile.local.ai-worker | 71 ------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 docker/Dockerfile.local.ai-worker diff --git a/docker/Dockerfile.local.ai-worker b/docker/Dockerfile.local.ai-worker deleted file mode 100644 index 922f20e8ca..0000000000 --- a/docker/Dockerfile.local.ai-worker +++ /dev/null @@ -1,71 +0,0 @@ -FROM --platform=$BUILDPLATFORM livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 as build - -ARG TARGETARCH -ARG BUILDARCH - -ENV GOARCH="$TARGETARCH" \ - PATH="/usr/local/go/bin:/go/bin:${PATH}" \ - PKG_CONFIG_PATH="/root/compiled/lib/pkgconfig" \ - CPATH="/usr/local/cuda_${TARGETARCH}/include" \ - LIBRARY_PATH="/usr/local/cuda_${TARGETARCH}/lib64" \ - DEBIAN_FRONTEND="noninteractive" \ - CGO_LDFLAGS="-L/usr/local/cuda_${TARGETARCH}/lib64" - -RUN apt update \ - && apt install -yqq software-properties-common curl apt-transport-https lsb-release yasm \ - && curl -fsSL https://dl.google.com/go/go1.21.5.linux-${BUILDARCH}.tar.gz | tar -C /usr/local -xz \ - && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ - && add-apt-repository "deb [arch=${BUILDARCH}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ - && curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ - && add-apt-repository "deb [arch=${BUILDARCH}] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \ - && apt update \ - && apt -yqq install clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python docker-ce-cli pciutils gcc-multilib libgcc-8-dev-arm64-cross gcc-mingw-w64-x86-64 zlib1g zlib1g-dev - -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \ - && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \ - && update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30 - -RUN GRPC_HEALTH_PROBE_VERSION=v0.3.6 \ - && curl -fsSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} -o /usr/bin/grpc_health_probe \ - && chmod +x /usr/bin/grpc_health_probe \ - && ldconfig /usr/local/lib - -ENV GOPATH=/go \ - GO_BUILD_DIR=/build/ \ - GOFLAGS="-mod=readonly" - -WORKDIR /src -COPY ./ai-worker ./ai-worker -RUN mkdir -p /go \ - && curl -fsSLO https://github.com/livepeer/livepeer-ml/releases/download/v0.3/tasmodel.pb - -COPY go-livepeer/install_ffmpeg.sh ./install_ffmpeg.sh - -ARG BUILD_TAGS -ENV BUILD_TAGS=${BUILD_TAGS} - -COPY ./go-livepeer/go.mod ./go-livepeer/go.sum ./ -RUN go mod download - -RUN ./install_ffmpeg.sh \ - && GO111MODULE=on go get -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 \ - && go get -v github.com/jstemmer/go-junit-report - -COPY ./go-livepeer . - -RUN make livepeer livepeer_cli livepeer_bench livepeer_router - -FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-amd64-base - -FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-arm64-base - -FROM livepeer-${TARGETARCH}-base - -ENV NVIDIA_DRIVER_CAPABILITIES=all - -COPY --from=build /build/ /usr/local/bin/ -COPY --from=build /usr/bin/grpc_health_probe /usr/local/bin/grpc_health_probe -COPY --from=build /src/tasmodel.pb /tasmodel.pb -COPY --from=build /usr/share/misc/pci.ids /usr/share/misc/pci.ids - -ENTRYPOINT ["/usr/local/bin/livepeer"] From cad5b6071614f598a5465570f80f440bfe3f4d95 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Fri, 5 Jul 2024 03:26:10 -0400 Subject: [PATCH 23/34] update go mod --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 9a3e93102a..328e27c640 100644 --- a/go.mod +++ b/go.mod @@ -207,6 +207,6 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 +replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e diff --git a/go.sum b/go.sum index e43ee48df4..dfe2f37a99 100644 --- a/go.sum +++ b/go.sum @@ -182,12 +182,10 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= - -github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50 h1:q0D+mEgnwRdK8MxTnB2qQ15YhSazp0EFN2BH6eud+ig= -github.com/eliteprox/ai-worker v0.0.0-20240704234154-f04146510a50/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= +github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e h1:KeX9oyrfxV63TQQPpr7dARppyzNkAcepwYScmfRh7Ns= github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= - github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From fa62b5a8460806e5d98f5bb2d54d47770dbde7bd Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Fri, 5 Jul 2024 03:27:57 -0400 Subject: [PATCH 24/34] remove comment --- server/ai_process.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/ai_process.go b/server/ai_process.go index ae6689b752..57a91345a9 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -526,7 +526,6 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface return nil, fmt.Errorf("unsupported request type %T", req) } - //var resp *worker.ImageResponse var resp interface{} cctx, cancel := context.WithTimeout(ctx, processingRetryTimeout) From bf03e0d348f0cb3dbf20c5f77a7919f4f5bba248 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Mon, 8 Jul 2024 11:01:02 -0400 Subject: [PATCH 25/34] update gomod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 328e27c640..112857eb63 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 -replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e +replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9 diff --git a/go.sum b/go.sum index dfe2f37a99..999b59d6c5 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e h1:KeX9oyrfxV63TQQPpr7dARppyzNkAcepwYScmfRh7Ns= -github.com/eliteprox/lpms v0.0.0-20240705040132-bad849896d4e/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= +github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9 h1:HHnrIZN8HeB9L9fXyBRkDArp8fZBidkNe/tCjrWG/xU= +github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From ff202d74caddde4a3ebc441f2b62ac446640b2de Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 10 Jul 2024 02:10:28 -0400 Subject: [PATCH 26/34] Update lpms mod --- common/util.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/util.go b/common/util.go index 74d2d9def2..1a6bf71510 100644 --- a/common/util.go +++ b/common/util.go @@ -549,7 +549,7 @@ func CalculateAudioDuration(audio types.File) (int64, error) { return 0, errors.New("Error getting codec info") } - duration := int64(mediaFormat.Dur) + duration := int64(mediaFormat.DurSecs) if duration <= 0 { return 0, ErrorCalculatingDuration } diff --git a/go.mod b/go.mod index 112857eb63..9950994708 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 -replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9 +replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1 diff --git a/go.sum b/go.sum index 999b59d6c5..2d1f3a3fe8 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9 h1:HHnrIZN8HeB9L9fXyBRkDArp8fZBidkNe/tCjrWG/xU= -github.com/eliteprox/lpms v0.0.0-20240708144644-e6389f243db9/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= +github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1 h1:cvm5dNBw0pMWKjV/nwihKvxBZEDxds4cDXnl3MlHlYU= +github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 8bad0fff04a6d25439fdfeb67db42f5c0253bd7d Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Wed, 10 Jul 2024 21:17:10 -0400 Subject: [PATCH 27/34] Update to latest lpms --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9950994708..bb6b6f8eda 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 -replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1 +replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876 diff --git a/go.sum b/go.sum index 2d1f3a3fe8..4f0b03081e 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1 h1:cvm5dNBw0pMWKjV/nwihKvxBZEDxds4cDXnl3MlHlYU= -github.com/eliteprox/lpms v0.0.0-20240710054834-d99d32b7b2d1/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= +github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876 h1:PzAjuprPa3CteTy32IiZR7BGGQXsd2QlK/v+NsWsYjM= +github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= From 2c0bfb995caf00060de80d25700725d5c848a5dd Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Thu, 11 Jul 2024 14:18:32 -0400 Subject: [PATCH 28/34] Update lpms --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index bb6b6f8eda..851c4d72a9 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/livepeer/ai-worker v0.0.8 github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 - github.com/livepeer/lpms v0.0.0-20240120150405-de94555cdc69 + github.com/livepeer/lpms v0.0.0-20240711175220-227325841434 github.com/livepeer/m3u8 v0.11.1 github.com/mattn/go-sqlite3 v1.14.18 github.com/oapi-codegen/nethttp-middleware v1.0.1 @@ -208,5 +208,3 @@ require ( ) replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 - -replace github.com/livepeer/lpms => github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876 diff --git a/go.sum b/go.sum index 4f0b03081e..e7e2a36dd5 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= -github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876 h1:PzAjuprPa3CteTy32IiZR7BGGQXsd2QlK/v+NsWsYjM= -github.com/eliteprox/lpms v0.0.0-20240711011429-1fd0f5cec876/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -541,6 +539,8 @@ github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cO github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded/go.mod h1:xkDdm+akniYxVT9KW1Y2Y7Hso6aW+rZObz3nrA9yTHw= github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 h1:4oH3NqV0NvcdS44Ld3zK2tO8IUiNozIggm74yobQeZg= github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18/go.mod h1:Jpf4jHK+fbWioBHRDRM1WadNT1qmY27g2YicTdO0Rtc= +github.com/livepeer/lpms v0.0.0-20240711175220-227325841434 h1:E7PKN6q/jMLapEV+eEwlwv87Xe5zacaVhvZ8T6AJR3c= +github.com/livepeer/lpms v0.0.0-20240711175220-227325841434/go.mod h1:Hr/JhxxPDipOVd4ZrGYWrdJfpVF8/SEI0nNr2ctAlkM= github.com/livepeer/m3u8 v0.11.1 h1:VkUJzfNTyjy9mqsgp5JPvouwna8wGZMvd/gAfT5FinU= github.com/livepeer/m3u8 v0.11.1/go.mod h1:IUqAtwWPAG2CblfQa4SVzTQoDcEMPyfNOaBSxqHMS04= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= From cb4360baca34c4678fb79e3455aca641d4e9c35a Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Sun, 14 Jul 2024 16:36:20 +0200 Subject: [PATCH 29/34] feat(ai): apply code improvements to AudioToText pipeline This commit applies several code improvements to the AudioToText codebase. --- cmd/livepeer/starter/starter.go | 1 - common/util.go | 10 +++++----- core/orchestrator.go | 8 +------- server/ai_http.go | 1 - server/ai_mediaserver.go | 18 +++++++++--------- server/ai_process.go | 30 ++++++++++++++---------------- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 945d0dee58..1495101ce6 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -625,7 +625,6 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { constraints[core.Capability_AudioToText].Models[config.ModelID] = modelConstraint n.SetBasePriceForCap("default", core.Capability_AudioToText, config.ModelID, big.NewRat(config.PricePerUnit, config.PixelsPerUnit)) - } if len(aiCaps) > 0 { diff --git a/common/util.go b/common/util.go index 1a6bf71510..46dae51aee 100644 --- a/common/util.go +++ b/common/util.go @@ -75,6 +75,9 @@ var ( ErrProfEncoder = fmt.Errorf("unknown VideoProfile encoder for protobufs") ErrProfName = fmt.Errorf("unknown VideoProfile profile name") + ErrUnsupportedAudioFormat = fmt.Errorf("audio format unsupported") + ErrAudioDurationCalculation = fmt.Errorf("audio duration calculation failed") + ext2mime = map[string]string{ ".ts": "video/mp2t", ".mp4": "video/mp4", @@ -532,10 +535,7 @@ func ParseEthAddr(strJsonKey string) (string, error) { return "", errors.New("Error parsing address from keyfile") } -// determines the duration of an mp3 audio file by reading the frames -var ErrUnsupportedFormat = errors.New("Unsupported audio file format") -var ErrorCalculatingDuration = errors.New("Error calculating duration") - +// CalculateAudioDuration calculates audio file duration using the lpms/ffmpeg package. func CalculateAudioDuration(audio types.File) (int64, error) { read, err := audio.Reader() if err != nil { @@ -551,7 +551,7 @@ func CalculateAudioDuration(audio types.File) (int64, error) { duration := int64(mediaFormat.DurSecs) if duration <= 0 { - return 0, ErrorCalculatingDuration + return 0, ErrAudioDurationCalculation } return duration, nil diff --git a/core/orchestrator.go b/core/orchestrator.go index 9505e97b30..4ceea94bba 100644 --- a/core/orchestrator.go +++ b/core/orchestrator.go @@ -950,13 +950,7 @@ func (n *LivepeerNode) upscale(ctx context.Context, req worker.UpscaleMultipartR } func (n *LivepeerNode) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { - - resp, err := n.AIWorker.AudioToText(ctx, req) - if err != nil { - return nil, err - } - - return resp, nil + return n.AIWorker.AudioToText(ctx, req) } func (n *LivepeerNode) imageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) { diff --git a/server/ai_http.go b/server/ai_http.go index b23bc61794..4ef6eac545 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -269,7 +269,6 @@ func handleAIRequest(ctx context.Context, w http.ResponseWriter, r *http.Request return } outPixels *= 1000 // Convert to milliseconds - default: respondWithError(w, "Unknown request type", http.StatusBadRequest) return diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 5ad5c6d865..29d33b9fed 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -340,29 +340,29 @@ func (ls *LivepeerServer) AudioToText() http.Handler { return } - clog.V(common.VERBOSE).Infof(ctx, "Received AudioToText request model_id=%v", *req.ModelId) + clog.V(common.VERBOSE).Infof(ctx, "Received AudioToText request audioSize=%v model_id=%v", req.Audio.FileSize(), *req.ModelId) params := aiRequestParams{ node: ls.LivepeerNode, - os: drivers.NodeStorage.NewSession(string(core.RandomManifestID())), + os: drivers.NodeStorage.NewSession(requestID), sessManager: ls.AISessionManager, } start := time.Now() resp, err := processAudioToText(ctx, params, req) if err != nil { - var e *ServiceUnavailableError - var reqError *BadRequestError - if errors.As(err, &e) { + var serviceUnavailableErr *ServiceUnavailableError + var badRequestErr *BadRequestError + if errors.As(err, &serviceUnavailableErr) { respondJsonError(ctx, w, err, http.StatusServiceUnavailable) return - } else if errors.As(err, &reqError) { + } + if errors.As(err, &badRequestErr) { respondJsonError(ctx, w, err, http.StatusBadRequest) return - } else { - respondJsonError(ctx, w, err, http.StatusInternalServerError) - return } + respondJsonError(ctx, w, err, http.StatusInternalServerError) + return } took := time.Since(start) diff --git a/server/ai_process.go b/server/ai_process.go index fdb3f8888f..49e27ab4ce 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/golang/glog" "github.com/livepeer/ai-worker/worker" "github.com/livepeer/go-livepeer/clog" "github.com/livepeer/go-livepeer/common" @@ -35,6 +34,10 @@ type ServiceUnavailableError struct { err error } +func (e *ServiceUnavailableError) Error() string { + return e.err.Error() +} + type BadRequestError struct { err error } @@ -43,10 +46,6 @@ func (e *BadRequestError) Error() string { return e.err.Error() } -func (e *ServiceUnavailableError) Error() string { - return e.err.Error() -} - type aiRequestParams struct { node *core.LivepeerNode os drivers.OSSession @@ -419,13 +418,13 @@ func submitAudioToText(ctx context.Context, params aiRequestParams, sess *AISess return nil, err } - outPixels, err := common.CalculateAudioDuration(req.Audio) + durationSeconds, err := common.CalculateAudioDuration(req.Audio) if err != nil { return nil, err } - glog.Infof("Submitting audio-to-text media with duration: %d seconds", outPixels) - outPixels *= 1000 // Convert to milliseconds - setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, outPixels) + + clog.V(common.VERBOSE).Infof(ctx, "Submitting audio-to-text media with duration: %d seconds", durationSeconds) + setHeaders, balUpdate, err := prepareAIPayment(ctx, sess, durationSeconds*1000) if err != nil { return nil, err } @@ -459,7 +458,7 @@ func submitAudioToText(ctx context.Context, params aiRequestParams, sess *AISess } // TODO: Refine this rough estimate in future iterations - sess.LatencyScore = took.Seconds() / float64(outPixels) + sess.LatencyScore = took.Seconds() / float64(durationSeconds) return &res, nil } @@ -505,7 +504,6 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface if v.ModelId != nil { modelID = *v.ModelId } - // Assuming submitImageToVideo returns a VideoResponse submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitImageToVideo(ctx, params, sess, v) } @@ -527,7 +525,6 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface submitFn = func(ctx context.Context, params aiRequestParams, sess *AISession) (interface{}, error) { return submitAudioToText(ctx, params, sess, v) } - // Add more cases as needed... default: return nil, fmt.Errorf("unsupported request type %T", req) } @@ -561,18 +558,19 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface params.sessManager.Complete(ctx, sess) break } - if errors.Is(err, common.ErrorCalculatingDuration) || errors.Is(err, common.ErrUnsupportedFormat) { - return nil, &BadRequestError{err} - } clog.Infof(ctx, "Error submitting request cap=%v modelID=%v try=%v orch=%v err=%v", cap, modelID, tries, sess.Transcoder(), err) params.sessManager.Remove(ctx, sess) + + if errors.Is(err, common.ErrAudioDurationCalculation) || errors.Is(err, common.ErrUnsupportedAudioFormat) { + return nil, &BadRequestError{err} + } } if resp == nil { return nil, &ServiceUnavailableError{err: errors.New("no orchestrators available")} } - return resp.(interface{}), nil + return resp, nil } func prepareAIPayment(ctx context.Context, sess *AISession, outPixels int64) (worker.RequestEditorFn, *BalanceUpdate, error) { From 5b24400c861b337ef88e2e3a91f0d7a35d6b4ed9 Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Mon, 15 Jul 2024 08:35:24 -0400 Subject: [PATCH 30/34] Remove unnecessary logic --- core/orchestrator.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/orchestrator.go b/core/orchestrator.go index 9505e97b30..86f98aa92a 100644 --- a/core/orchestrator.go +++ b/core/orchestrator.go @@ -952,11 +952,7 @@ func (n *LivepeerNode) upscale(ctx context.Context, req worker.UpscaleMultipartR func (n *LivepeerNode) AudioToText(ctx context.Context, req worker.AudioToTextMultipartRequestBody) (*worker.TextResponse, error) { resp, err := n.AIWorker.AudioToText(ctx, req) - if err != nil { - return nil, err - } - - return resp, nil + return resp, err } func (n *LivepeerNode) imageToVideo(ctx context.Context, req worker.ImageToVideoMultipartRequestBody) (*worker.ImageResponse, error) { From c131e5946f9ee3a0f29733508e3ef90e83a1093b Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Mon, 15 Jul 2024 14:56:11 -0400 Subject: [PATCH 31/34] Remove unused error --- common/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/util.go b/common/util.go index 46dae51aee..8a9eb15219 100644 --- a/common/util.go +++ b/common/util.go @@ -75,7 +75,6 @@ var ( ErrProfEncoder = fmt.Errorf("unknown VideoProfile encoder for protobufs") ErrProfName = fmt.Errorf("unknown VideoProfile profile name") - ErrUnsupportedAudioFormat = fmt.Errorf("audio format unsupported") ErrAudioDurationCalculation = fmt.Errorf("audio duration calculation failed") ext2mime = map[string]string{ From cc10b98804c0f4798772d0b5dc03e064b238cb5d Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Mon, 15 Jul 2024 16:00:22 -0400 Subject: [PATCH 32/34] Fix missing err --- server/ai_process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ai_process.go b/server/ai_process.go index 49e27ab4ce..be8571dc26 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -562,7 +562,7 @@ func processAIRequest(ctx context.Context, params aiRequestParams, req interface clog.Infof(ctx, "Error submitting request cap=%v modelID=%v try=%v orch=%v err=%v", cap, modelID, tries, sess.Transcoder(), err) params.sessManager.Remove(ctx, sess) - if errors.Is(err, common.ErrAudioDurationCalculation) || errors.Is(err, common.ErrUnsupportedAudioFormat) { + if errors.Is(err, common.ErrAudioDurationCalculation) { return nil, &BadRequestError{err} } } From 6fabbf277504a9e459343f503ba537b5c28ea30a Mon Sep 17 00:00:00 2001 From: Elite Encoder Date: Tue, 16 Jul 2024 10:19:20 -0400 Subject: [PATCH 33/34] Update go.mod and tidy --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 851c4d72a9..e9bc73da7d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/jaypipes/ghw v0.10.0 github.com/jaypipes/pcidb v1.0.0 - github.com/livepeer/ai-worker v0.0.8 + github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 github.com/livepeer/lpms v0.0.0-20240711175220-227325841434 @@ -206,5 +206,3 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -replace github.com/livepeer/ai-worker => github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 diff --git a/go.sum b/go.sum index e7e2a36dd5..37a16bce54 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,6 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12 h1:LCnOJCD97i2jmR7gY5gJieYLS+01XHnbWkCpmHC1t28= -github.com/eliteprox/ai-worker v0.0.0-20240705062703-0908b518eb12/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -533,6 +531,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= +github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d h1:qGDGo50Ya76lxSttItQRf+KMjKOy3CH6FSoG/XD/3UE= +github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA= From 0cf872b58ddb2257cc1327c60d8f5b913e1bbb28 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Wed, 17 Jul 2024 15:28:32 +0200 Subject: [PATCH 34/34] chore(ai): update ai-worker and lpms to latest version This commit ensures that the ai-worker and lpms are at the latest versions which contain the changes needed for the audio-to-text pipeline. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9bc73da7d..c80fe64f37 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/jaypipes/ghw v0.10.0 github.com/jaypipes/pcidb v1.0.0 - github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d + github.com/livepeer/ai-worker v0.1.0 github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b github.com/livepeer/livepeer-data v0.7.5-0.20231004073737-06f1f383fb18 github.com/livepeer/lpms v0.0.0-20240711175220-227325841434 diff --git a/go.sum b/go.sum index 37a16bce54..74df1b7267 100644 --- a/go.sum +++ b/go.sum @@ -531,8 +531,8 @@ github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4n github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= -github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d h1:qGDGo50Ya76lxSttItQRf+KMjKOy3CH6FSoG/XD/3UE= -github.com/livepeer/ai-worker v0.0.9-0.20240716140832-9fc476ebb69d/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= +github.com/livepeer/ai-worker v0.1.0 h1:SJBZuxeK0vEzJPBzf5osdgVCxHYZt7ZKR2CvZ7Q7iog= +github.com/livepeer/ai-worker v0.1.0/go.mod h1:Xlnb0nFG2VsGeMG9hZmReVQXeFt0Dv28ODiUT2ooyLE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b h1:VQcnrqtCA2UROp7q8ljkh2XA/u0KRgVv0S1xoUvOweE= github.com/livepeer/go-tools v0.3.6-0.20240130205227-92479de8531b/go.mod h1:hwJ5DKhl+pTanFWl+EUpw1H7ukPO/H+MFpgA7jjshzw= github.com/livepeer/joy4 v0.1.2-0.20191121080656-b2fea45cbded h1:ZQlvR5RB4nfT+cOQee+WqmaDOgGtP2oDMhcVvR4L0yA=