Merge pull request #328 from Zibbp/queue-temporal

feat: temporal queue

.devcontainer/Dockerfile

@@ -1,6 +1,6 @@
 FROM mcr.microsoft.com/devcontainers/go:1-bullseye
 
-ENV CHAT_DOWNLOADER_VER=0.2.4
+ENV CHAT_DOWNLOADER_VER=0.2.8
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
   && apt-get -y install --no-install-recommends ffmpeg python3 python3-pip \
@@ -13,7 +13,7 @@ WORKDIR /tmp
 
 RUN wget https://github.com/rsms/inter/releases/download/v4.0-beta7/Inter-4.0-beta7.zip && unzip Inter-4.0-beta7.zip && mkdir -p /usr/share/fonts/opentype/inter/ && cp /tmp/Desktop/Inter-*.otf /usr/share/fonts/opentype/inter/ && fc-cache -f -v
 
-RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.2/TwitchDownloaderCLI-1.53.2-Linux-x64.zip && unzip TwitchDownloaderCLI-1.53.2-Linux-x64.zip && mv TwitchDownloaderCLI /usr/local/bin/ && chmod +x /usr/local/bin/TwitchDownloaderCLI && rm TwitchDownloaderCLI-1.53.2-Linux-x64.zip
+RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.6/TwitchDownloaderCLI-1.53.6-Linux-x64.zip && unzip TwitchDownloaderCLI-1.53.6-Linux-x64.zip && mv TwitchDownloaderCLI /usr/local/bin/ && chmod +x /usr/local/bin/TwitchDownloaderCLI && rm TwitchDownloaderCLI-1.53.6-Linux-x64.zip
 
 #RUN wget https://github.com/xenova/chat-downloader/archive/refs/tags/v${CHAT_DOWNLOADER_VER}.tar.gz
 #RUN tar -xvf v${CHAT_DOWNLOADER_VER}.tar.gz && cd chat-downloader-${CHAT_DOWNLOADER_VER} && python3 setup.py install && cd .. && rm -f v${CHAT_DOWNLOADER_VER}.tar.gz && rm -rf chat-downloader-${CHAT_DOWNLOADER_VER}

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
   "build": { "dockerfile": "Dockerfile" },
   "features": {
-    "ghcr.io/eitsupi/devcontainer-features/go-task:1": {}
+    "ghcr.io/jungaretti/features/make:1": {}
   },
   "customizations": {
     "vscode": {

.gitignore

@@ -24,4 +24,5 @@ go.work
 
 /cmd/server/__debug_bin
 dev
-tmp
+tmp
+**/__debug

.vscode/launch.json

@@ -3,14 +3,28 @@
   // Hover to view descriptions of existing attributes.
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
+  "compounds": [
+    {
+      "name": "server/worker",
+      "configurations": ["dev-server", "dev-worker"]
+    }
+  ],
   "configurations": [
     {
-      "name": "Launch Package",
+      "name": "dev-server",
       "type": "go",
       "request": "launch",
       "mode": "auto",
       "program": "${workspaceFolder}/cmd/server/main.go",
       "envFile": "${workspaceFolder}/.env.dev"
+    },
+    {
+      "name": "dev-worker",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/cmd/worker/main.go",
+      "envFile": "${workspaceFolder}/.env.dev"
     }
   ]
 }

Dockerfile

@@ -1,27 +1,27 @@
-FROM golang:1.20 AS build-stage-01
+FROM golang:1.21 AS build-stage-01
 
 RUN mkdir /app
 ADD . /app
 WORKDIR /app
 
 RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -X main.Version=${VERSION} -X main.BuildTime=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X main.GitHash=`git rev-parse HEAD`" -o ganymede-api cmd/server/main.go
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -X main.Version=${VERSION} -X main.BuildTime=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X main.GitHash=`git rev-parse HEAD`" -o ganymede-worker cmd/worker/main.go
 
 FROM alpine:latest AS build-stage-02
 
 RUN apk add --update --no-cache unzip git
 
 WORKDIR /tmp
 RUN wget https://github.com/rsms/inter/releases/download/v3.19/Inter-3.19.zip && unzip Inter-3.19.zip
-RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.2/TwitchDownloaderCLI-1.53.2-LinuxAlpine-x64.zip && unzip TwitchDownloaderCLI-1.53.2-LinuxAlpine-x64.zip
+RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.6/TwitchDownloaderCLI-1.53.6-LinuxAlpine-x64.zip && unzip TwitchDownloaderCLI-1.53.6-LinuxAlpine-x64.zip
 
 RUN git clone https://github.com/xenova/chat-downloader.git
 
 FROM alpine:latest AS production
 
 # install packages
-RUN apk add --update --no-cache python3 fontconfig icu-libs python3-dev gcc g++ ffmpeg bash tzdata shadow su-exec && ln -sf python3 /usr/bin/python
-RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip streamlink
+RUN apk add --update --no-cache python3 fontconfig icu-libs python3-dev gcc g++ ffmpeg bash tzdata shadow su-exec py3-pip && ln -sf python3 /usr/bin/python
+RUN pip3 install --no-cache --upgrade pip streamlink --break-system-packages
 
 # setup user
 RUN groupmod -g 1000 users && \
@@ -50,6 +50,7 @@ RUN chmod +x /usr/local/bin/TwitchDownloaderCLI
 WORKDIR /opt/app
 
 COPY --from=build-stage-01 /app/ganymede-api .
+COPY --from=build-stage-01 /app/ganymede-worker .
 
 EXPOSE 4000
 

Dockerfile.aarch64

@@ -1,10 +1,11 @@
-FROM arm64v8/golang:1.20 AS build-stage-01
+FROM arm64v8/golang:1.21 AS build-stage-01
 
 RUN mkdir /app
 ADD . /app
 WORKDIR /app
 
 RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -X main.Version=${VERSION} -X main.BuildTime=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X main.GitHash=`git rev-parse HEAD`" -o ganymede-api cmd/server/main.go
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -X main.Version=${VERSION} -X main.BuildTime=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X main.GitHash=`git rev-parse HEAD`" -o ganymede-worker cmd/worker/main.go
 
 FROM arm64v8/debian:bullseye AS build-stage-02
 
@@ -13,7 +14,7 @@ RUN apt-get install unzip wget git -y
 
 WORKDIR /tmp
 RUN wget https://github.com/rsms/inter/releases/download/v3.19/Inter-3.19.zip && unzip Inter-3.19.zip
-RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.2/TwitchDownloaderCLI-1.53.2-LinuxArm.zip && unzip TwitchDownloaderCLI-1.53.2-LinuxArm.zip
+RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.53.6/TwitchDownloaderCLI-1.53.6-LinuxArm.zip && unzip TwitchDownloaderCLI-1.53.6-LinuxArm.zip
 
 RUN git clone https://github.com/xenova/chat-downloader.git
 
@@ -24,7 +25,7 @@ RUN dpkg --add-architecture armhf
 
 RUN apt-get update
 RUN apt-get install python3 python3-pip fontconfig icu-devtools python3-dev gcc libc-dev curl g++ ffmpeg bash tzdata -y
-RUN pip3 install --no-cache --upgrade pip streamlink
+RUN pip3 install --no-cache --upgrade pip streamlink --break-system-packages
 
 ## Installing su-exec in debain/ubuntu container.
 RUN  set -ex; \
@@ -70,6 +71,7 @@ RUN chmod +x /usr/local/bin/TwitchDownloaderCLI
 WORKDIR /opt/app
 
 COPY --from=build-stage-01 /app/ganymede-api .
+COPY --from=build-stage-01 /app/ganymede-worker .
 
 EXPOSE 4000
 

Makefile

@@ -0,0 +1,11 @@
+prereq_command:
+	set -o allexport; source .env.dev; set +o allexport;
+
+dev_server: prereq_command
+	go run cmd/server/main.go
+
+dev_worker: prereq_command
+	go run cmd/worker/main.go
+
+ent_generate:
+	go run -mod=mod entgo.io/ent/cmd/ent generate --feature sql/upsert ./ent/schema

cmd/server/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"strconv"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/zibbp/ganymede/internal/archive"
 	"github.com/zibbp/ganymede/internal/auth"
 	"github.com/zibbp/ganymede/internal/channel"
+	"github.com/zibbp/ganymede/internal/chapter"
 	"github.com/zibbp/ganymede/internal/config"
 	"github.com/zibbp/ganymede/internal/database"
 	"github.com/zibbp/ganymede/internal/kv"
@@ -24,6 +26,7 @@ import (
 	"github.com/zibbp/ganymede/internal/queue"
 	"github.com/zibbp/ganymede/internal/scheduler"
 	"github.com/zibbp/ganymede/internal/task"
+	"github.com/zibbp/ganymede/internal/temporal"
 	transportHttp "github.com/zibbp/ganymede/internal/transport/http"
 	"github.com/zibbp/ganymede/internal/twitch"
 	"github.com/zibbp/ganymede/internal/user"
@@ -75,9 +78,12 @@ func Run() error {
 	//	log.Error().Err(err).Msg("failed to create database connection")
 	//	return err
 	//}
-	database.InitializeDatabase()
+	database.InitializeDatabase(false)
 	store := database.DB()
 
+	// Initialize temporal client
+	temporal.InitializeTemporalClient()
+
 	authService := auth.NewService(store)
 	channelService := channel.NewService(store)
 	vodService := vod.NewService(store)
@@ -93,8 +99,9 @@ func Run() error {
 	metricsService := metrics.NewService(store)
 	playlistService := playlist.NewService(store)
 	taskService := task.NewService(store, liveService, archiveService)
+	chapterService := chapter.NewService()
 
-	httpHandler := transportHttp.NewHandler(authService, channelService, vodService, queueService, twitchService, archiveService, adminService, userService, configService, liveService, schedulerService, playbackService, metricsService, playlistService, taskService)
+	httpHandler := transportHttp.NewHandler(authService, channelService, vodService, queueService, twitchService, archiveService, adminService, userService, configService, liveService, schedulerService, playbackService, metricsService, playlistService, taskService, chapterService)
 
 	if err := httpHandler.Serve(); err != nil {
 		return err
@@ -104,6 +111,9 @@ func Run() error {
 }
 
 func main() {
+	if os.Getenv("ENV") == "dev" {
+		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+	}
 	kv.DB().Set("version", Version)
 	kv.DB().Set("build_time", BuildTime)
 	kv.DB().Set("git_hash", GitHash)