From 399927b7cc6ba46bd528e3e782398070363a9253 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sun, 28 May 2023 22:14:56 +0300 Subject: [PATCH 01/16] Plugin: Upscale plugin based on Real-ESRGAN --- .github/workflows/plugins-docker.yaml | 24 +++++++ ext/realesrgan/.gitignore | 1 + ext/realesrgan/Dockerfile | 13 ++++ ext/realesrgan/main.py | 90 +++++++++++++++++++++++++++ ext/realesrgan/requirements.txt | 2 + 5 files changed, 130 insertions(+) create mode 100644 .github/workflows/plugins-docker.yaml create mode 100644 ext/realesrgan/.gitignore create mode 100644 ext/realesrgan/Dockerfile create mode 100644 ext/realesrgan/main.py create mode 100644 ext/realesrgan/requirements.txt diff --git a/.github/workflows/plugins-docker.yaml b/.github/workflows/plugins-docker.yaml new file mode 100644 index 00000000000..81e8b06e074 --- /dev/null +++ b/.github/workflows/plugins-docker.yaml @@ -0,0 +1,24 @@ +on: + pull_request: + paths: + - "ext/**" + +jobs: + publish-hello-docker-image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: ext/realesrgan + tags: kvalev/realesrgan:0.3.0 + push: true diff --git a/ext/realesrgan/.gitignore b/ext/realesrgan/.gitignore new file mode 100644 index 00000000000..05424f2a4c8 --- /dev/null +++ b/ext/realesrgan/.gitignore @@ -0,0 +1 @@ +weights diff --git a/ext/realesrgan/Dockerfile b/ext/realesrgan/Dockerfile new file mode 100644 index 00000000000..6bcd9bc65c4 --- /dev/null +++ b/ext/realesrgan/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10.11-slim-buster + +RUN apt-get update && apt-get install -y libgl1 libglib2.0-0 + +COPY requirements.txt /app/ +RUN pip install -r /app/requirements.txt + +COPY . /app/ +WORKDIR /app + +EXPOSE 5001 + +CMD ["python3", "main.py"] diff --git a/ext/realesrgan/main.py b/ext/realesrgan/main.py new file mode 100644 index 00000000000..3d026e0776b --- /dev/null +++ b/ext/realesrgan/main.py @@ -0,0 +1,90 @@ +import base64 +from io import BytesIO +from basicsr.archs.rrdbnet_arch import RRDBNet +from basicsr.utils.download_util import load_file_from_url +from flask import Flask, request +from realesrgan import RealESRGANer +from realesrgan.archs.srvgg_arch import SRVGGNetCompact +from PIL import Image +import numpy as np +import torch + +import logging +import os + +app = Flask(__name__) + +REAL_ESRGAN_MODELS = { + "realesr-general-x4v3": { + "url": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth", + "model_md5": "91a7644643c884ee00737db24e478156", + "scale": 4, + "model": lambda: SRVGGNetCompact( + num_in_ch=3, + num_out_ch=3, + num_feat=64, + num_conv=32, + upscale=4, + act_type="prelu", + ), + }, + "RealESRGAN_x4plus": { + "url": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth", + "model_md5": "99ec365d4afad750833258a1a24f44ca", + "scale": 4, + "model": lambda: RRDBNet( + num_in_ch=3, + num_out_ch=3, + num_feat=64, + num_block=23, + num_grow_ch=32, + scale=4, + ), + }, +} + +name = "realesr-general-x4v3" + +if name not in REAL_ESRGAN_MODELS: + raise ValueError(f"Unknown RealESRGAN model name: {name}") + +model_info = REAL_ESRGAN_MODELS[name] + +model_path = os.path.join('weights', name + '.pth') +if not os.path.isfile(model_path): + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + model_path = load_file_from_url(url=model_info["url"], model_dir=os.path.join(ROOT_DIR, 'weights'), progress=True, file_name=None) +logging.info(f"RealESRGAN model path: {model_path}") + +device = "cuda" if torch.cuda.is_available() else "cpu" + +model = RealESRGANer( + scale=model_info["scale"], + model_path=model_path, + model=model_info["model"](), + half=True if "cuda" in device else False, + tile=512, + tile_pad=10, + pre_pad=10, + device=device, +) + +@app.route('/superscale', methods=['POST']) +def superscale(): + scale = request.json.get("scale", model_info["scale"]) + + image_data = base64.b64decode(request.json['image']) + image = np.asarray(Image.open(BytesIO(image_data))) + print(f"RealESRGAN input shape: {image.shape}, scale: {scale}", flush=True) + + upsampled = model.enhance(image, outscale=scale)[0] + upsampled_img = Image.fromarray(upsampled) + print(f"RealESRGAN output shape: {upsampled.shape}", flush=True) + + with BytesIO() as buffer: + upsampled_img.save(buffer, format="jpeg") + return {"image": base64.b64encode(buffer.getvalue()).decode()} + +if __name__ == '__main__': + print("running") + app.run(host='0.0.0.0', port=5001) diff --git a/ext/realesrgan/requirements.txt b/ext/realesrgan/requirements.txt new file mode 100644 index 00000000000..b1a9b1dda23 --- /dev/null +++ b/ext/realesrgan/requirements.txt @@ -0,0 +1,2 @@ +realesrgan==0.3.0 +flask==2.3.2 From 9a8343d541038ced1847477194d65b43b88e9186 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Mon, 29 May 2023 17:17:21 +0300 Subject: [PATCH 02/16] Plugin: Predict plugin based on YOLO8 --- .github/workflows/plugins-docker.yaml | 18 +++++++++++++++++- ext/yolo8/Dockerfile | 13 +++++++++++++ ext/yolo8/main.py | 25 +++++++++++++++++++++++++ ext/yolo8/requirements.txt | 2 ++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 ext/yolo8/Dockerfile create mode 100644 ext/yolo8/main.py create mode 100644 ext/yolo8/requirements.txt diff --git a/.github/workflows/plugins-docker.yaml b/.github/workflows/plugins-docker.yaml index 81e8b06e074..6c703e4a96f 100644 --- a/.github/workflows/plugins-docker.yaml +++ b/.github/workflows/plugins-docker.yaml @@ -4,19 +4,35 @@ on: - "ext/**" jobs: - publish-hello-docker-image: + publish-plugin-docker-images: + strategy: + fail-fast: false + matrix: + plugins: + - realesrgan + - yolo8 + runs-on: ubuntu-latest steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + changed: 'ext/${{ matrix.plugins }}/**' + - name: Checkout + if: steps.changes.outputs.changed == 'true' uses: actions/checkout@v3 - name: Log in to Docker Hub + if: steps.changes.outputs.changed == 'true' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push Docker image + if: steps.changes.outputs.changed == 'true' uses: docker/build-push-action@v2 with: context: ext/realesrgan diff --git a/ext/yolo8/Dockerfile b/ext/yolo8/Dockerfile new file mode 100644 index 00000000000..0e03ca5a127 --- /dev/null +++ b/ext/yolo8/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10.11-slim-buster + +RUN apt-get update && apt-get install -y libgl1 libglib2.0-0 + +COPY requirements.txt /app/ +RUN pip install -r /app/requirements.txt + +COPY . /app/ +WORKDIR /app + +EXPOSE 5000 + +CMD ["python3", "main.py"] diff --git a/ext/yolo8/main.py b/ext/yolo8/main.py new file mode 100644 index 00000000000..f59633f3597 --- /dev/null +++ b/ext/yolo8/main.py @@ -0,0 +1,25 @@ +import base64 +from flask import Flask, request +from io import BytesIO +from PIL import Image +from ultralytics import YOLO +from ultralytics.yolo.engine.results import Results + +app = Flask(__name__) +model = YOLO("yolov8n.pt") + +@app.route('/hello', methods=['GET']) +def hello(): + return "elloh" + +@app.route('/predict', methods=['POST']) +def predict(): + image_data = base64.b64decode(request.json['image']) + image = Image.open(BytesIO(image_data)) + + results: Results = model.predict(image) + + return results[0].tojson() + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000) diff --git a/ext/yolo8/requirements.txt b/ext/yolo8/requirements.txt new file mode 100644 index 00000000000..26023f3205a --- /dev/null +++ b/ext/yolo8/requirements.txt @@ -0,0 +1,2 @@ +Flask +ultralytics From fd0c6c65ec910a9e3cca28f22790028d1af88522 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Mon, 29 May 2023 17:39:08 +0300 Subject: [PATCH 03/16] Plugins: Properly versioning plugin docker images --- .github/workflows/plugins-docker.yaml | 16 ++++++++++------ ext/yolo8/requirements.txt | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/plugins-docker.yaml b/.github/workflows/plugins-docker.yaml index 6c703e4a96f..a16c4955cfd 100644 --- a/.github/workflows/plugins-docker.yaml +++ b/.github/workflows/plugins-docker.yaml @@ -1,3 +1,5 @@ +name: Build plugin docker images + on: pull_request: paths: @@ -8,9 +10,11 @@ jobs: strategy: fail-fast: false matrix: - plugins: - - realesrgan - - yolo8 + plugin: + - name: realesrgan + version: 0.3.0 + - name: yolo8 + version: 8.0.110 runs-on: ubuntu-latest steps: @@ -18,7 +22,7 @@ jobs: id: changes with: filters: | - changed: 'ext/${{ matrix.plugins }}/**' + changed: 'ext/${{ matrix.plugin.name }}/**' - name: Checkout if: steps.changes.outputs.changed == 'true' @@ -35,6 +39,6 @@ jobs: if: steps.changes.outputs.changed == 'true' uses: docker/build-push-action@v2 with: - context: ext/realesrgan - tags: kvalev/realesrgan:0.3.0 + context: ext/${{ matrix.plugin.name }} + tags: kvalev/${{ matrix.plugin.name }}:${{ matrix.plugin.version }} push: true diff --git a/ext/yolo8/requirements.txt b/ext/yolo8/requirements.txt index 26023f3205a..d07ca6e7c54 100644 --- a/ext/yolo8/requirements.txt +++ b/ext/yolo8/requirements.txt @@ -1,2 +1,2 @@ -Flask -ultralytics +ultralytics==8.0.110 +flask==2.3.2 From cd1eea687fe079c538b289b2d8b4425afd0c93d2 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Wed, 31 May 2023 01:47:38 +0300 Subject: [PATCH 04/16] Docker: Compose file with PhotoPrism plugins --- docker-compose.plugins.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docker-compose.plugins.yml diff --git a/docker-compose.plugins.yml b/docker-compose.plugins.yml new file mode 100644 index 00000000000..664931b1e4b --- /dev/null +++ b/docker-compose.plugins.yml @@ -0,0 +1,22 @@ +version: '3.5' + +## FOR TEST AND DEVELOPMENT ONLY, DO NOT USE IN PRODUCTION ## +## Setup: https://docs.photoprism.app/developer-guide/setup/ ## + +services: + ## PhotoPrism Development Environment + photoprism: + environment: + PHOTOPRISM_PLUGIN_YOLO8_ENABLED: "true" + PHOTOPRISM_PLUGIN_YOLO8_HOSTNAME: "yolo8" + PHOTOPRISM_PLUGIN_YOLO8_PORT: "5000" + + ## Image classification API + yolo8: + image: kvalev/yolo8:8.0.110 + pull_policy: always + + ## Image upscaling API + realesrgan: + image: kvalev/realesrgan:0.3.0 + pull_policy: always From 285378f6f692ee662d54036fdaa1a8bd41f5e31d Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Wed, 31 May 2023 01:48:25 +0300 Subject: [PATCH 05/16] Docker: Add phpmyadmin to main compose file This will simplify browsing database data during development. --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3de6309a7f6..e13ae51ed56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,6 +149,17 @@ services: MARIADB_PASSWORD: "photoprism" MARIADB_ROOT_PASSWORD: "photoprism" + phpmyadmin: + image: phpmyadmin/phpmyadmin:5.0 + restart: always + environment: + PMA_HOST: mariadb + PMA_PORT: 4001 + ports: + - 6060:80 + depends_on: + - mariadb + ## HTTPS Reverse Proxy (recommended) ## ## includes "*.localssl.dev" SSL certificate for test environments ## Docs: https://doc.traefik.io/traefik/ From f7294301c5749b9905f1b2dd6147d881573a71f7 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Wed, 31 May 2023 01:49:18 +0300 Subject: [PATCH 06/16] Docker: Reduce PhotoPrism workers to 1 --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index e13ae51ed56..05c178376cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,6 +106,7 @@ services: TF_CPP_MIN_LOG_LEVEL: 0 # show TensorFlow log messages for development ## Run/install on first startup (options: update https gpu tensorflow davfs clitools clean): PHOTOPRISM_INIT: "https tensorflow" + PHOTOPRISM_WORKERS: 1 ## Hardware Video Transcoding (optional): # PHOTOPRISM_FFMPEG_ENCODER: "nvidia" # FFmpeg encoder ("software", "intel", "nvidia", "apple", "raspberry", "vaapi") Intel: "intel" for Broadwell or later and "vaapi" for Haswell or earlier # PHOTOPRISM_FFMPEG_ENCODER: "intel" # FFmpeg encoder ("software", "intel", "nvidia", "apple", "raspberry", "vaapi") Intel: "intel" for Broadwell or later and "vaapi" for Haswell or earlier` From 2804721cb9afda41cab4eb6b07e6cabac6d1fadf Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Wed, 31 May 2023 15:52:40 +0300 Subject: [PATCH 07/16] Plugin: Classification and detection endpoints in YOLO8 plugin docker --- ext/yolo8/main.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ext/yolo8/main.py b/ext/yolo8/main.py index f59633f3597..068f59a2013 100644 --- a/ext/yolo8/main.py +++ b/ext/yolo8/main.py @@ -2,22 +2,40 @@ from flask import Flask, request from io import BytesIO from PIL import Image +from typing import List from ultralytics import YOLO from ultralytics.yolo.engine.results import Results app = Flask(__name__) -model = YOLO("yolov8n.pt") + +model = "yolov8n" +detect_model = YOLO(f"{model}.pt") +classify_model = YOLO(f"{model}-cls.pt") @app.route('/hello', methods=['GET']) def hello(): return "elloh" -@app.route('/predict', methods=['POST']) -def predict(): +@app.route('/classify', methods=['POST']) +def classify(): + image_data = base64.b64decode(request.json['image']) + image = Image.open(BytesIO(image_data)) + + results: List[Results] = classify_model.predict(image) + result = results[0] + + # take only the top3 results + take = min(len(result.names), 3) + top_n_idx = result.probs.argsort(0, descending=True)[:take].tolist() + + return {result.names[idx]: result.probs[idx].item() for idx in top_n_idx} + +@app.route('/detect', methods=['POST']) +def detect(): image_data = base64.b64decode(request.json['image']) image = Image.open(BytesIO(image_data)) - results: Results = model.predict(image) + results: List[Results] = detect_model.predict(image) return results[0].tojson() From 991e71bff9d24bb34da497ea2d05cd53615587f3 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Thu, 1 Jun 2023 00:54:05 +0300 Subject: [PATCH 08/16] Plugin: Improve demo plugin code style --- internal/plugin/demo/demo.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/plugin/demo/demo.go b/internal/plugin/demo/demo.go index 70a973452a0..90d5c362388 100644 --- a/internal/plugin/demo/demo.go +++ b/internal/plugin/demo/demo.go @@ -11,11 +11,11 @@ func (p DemoPlugin) Name() string { return "demo" } -func (DemoPlugin) Configure(plugin.PluginConfig) error { +func (*DemoPlugin) Configure(plugin.PluginConfig) error { return nil } -func (DemoPlugin) OnIndex(file *entity.File, photo *entity.Photo) error { +func (*DemoPlugin) OnIndex(file *entity.File, photo *entity.Photo) error { photo.Details.Notes = "hello from demo plugin" photo.Details.NotesSrc = entity.SrcManual @@ -24,4 +24,4 @@ func (DemoPlugin) OnIndex(file *entity.File, photo *entity.Photo) error { // Export the plugin. -var Plugin DemoPlugin \ No newline at end of file +var Plugin DemoPlugin From c3836e6a02f45338276b3ce33c38e8503c17c07b Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Thu, 1 Jun 2023 01:00:00 +0300 Subject: [PATCH 09/16] Plugin: Classification plugin based on Yolo8 --- Makefile | 13 ++- internal/plugin/yolo8/yolo8.go | 204 +++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 internal/plugin/yolo8/yolo8.go diff --git a/Makefile b/Makefile index e21f8684b6a..596e6ca610b 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,9 @@ INSTALL_USER ?= $(DESTUID):$(DESTGID) INSTALL_MODE ?= u+rwX,a+rX INSTALL_MODE_BIN ?= 755 +# Other parameters. +PLUGINS_PATH ?= storage/plugins + UID := $(shell id -u) GID := $(shell id -g) HASRICHGO := $(shell which richgo) @@ -179,6 +182,8 @@ clean-local-cache: rm -rf $(BUILD_PATH)/storage/cache/* clean-local-config: rm -f $(BUILD_PATH)/config/* +clean-plugins: + rm -f $(PLUGINS_PATH)/* dep-list: go list -u -m -json all | go-mod-outdated -direct dep-npm: @@ -219,9 +224,13 @@ build-race: build-static: rm -f $(BINARY_NAME) scripts/build.sh static $(BINARY_NAME) +build-plugins: build-plugin-demo build-plugin-yolo8 build-plugin-demo: - mkdir -p storage/plugins - go build -buildmode=plugin -o storage/plugins/demo.so internal/plugin/demo/demo.go + mkdir -p $(PLUGINS_PATH) + go build -tags=debug -buildmode=plugin -o $(PLUGINS_PATH)/demo.so internal/plugin/demo/demo.go +build-plugin-yolo8: + mkdir -p $(PLUGINS_PATH) + go build -tags=debug -buildmode=plugin -o $(PLUGINS_PATH)/yolo8.so internal/plugin/yolo8/yolo8.go build-tensorflow: docker build -t photoprism/tensorflow:build docker/tensorflow docker run -ti photoprism/tensorflow:build bash diff --git a/internal/plugin/yolo8/yolo8.go b/internal/plugin/yolo8/yolo8.go new file mode 100644 index 00000000000..c6c3ca3795d --- /dev/null +++ b/internal/plugin/yolo8/yolo8.go @@ -0,0 +1,204 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + "time" + + "github.com/photoprism/photoprism/internal/classify" + "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/photoprism" + "github.com/photoprism/photoprism/internal/plugin" + "github.com/photoprism/photoprism/pkg/clean" + "github.com/photoprism/photoprism/pkg/fs" +) + +type ClassifyResults map[string]float64 +type DetectResults []DetectResult + +type DetectResult struct { + Name string `json:"name"` + Class int16 `json:"class"` + Confidence float64 `json:"confidence"` +} + +type Yolo8Plugin struct { + hostname string + port string + confThreshold float64 +} + +func (p Yolo8Plugin) Name() string { + return "yolo8" +} + +func (p *Yolo8Plugin) Configure(config plugin.PluginConfig) error { + hostname, ok := config["hostname"] + if !ok { + return fmt.Errorf("hostname parameter is mandatory") + } + + port, ok := config["port"] + if !ok { + return fmt.Errorf("port parameter is mandatory") + } + + threshold := 0.5 + var err error + + if t, ok := config["confidence_threshold"]; ok { + if threshold, err = strconv.ParseFloat(t, 64); err != nil { + return err + } + } + + p.hostname = hostname + p.port = port + p.confThreshold = threshold + + return nil +} + +func (p *Yolo8Plugin) OnIndex(file *entity.File, photo *entity.Photo) error { + image, err := p.image(file) + if err != nil { + return err + } + + labels, err := p.classify(image) + if err != nil { + return err + } + + photo.AddLabels(labels) + fmt.Printf("adding labels %#v", labels) + + return nil +} + +func (p *Yolo8Plugin) image(f *entity.File) (string, error) { + filePath := photoprism.FileName(f.FileRoot, f.FileName) + + if !fs.FileExists(filePath) { + return "", fmt.Errorf("file %s is missing", clean.Log(f.FileName)) + } + + data, err := ioutil.ReadFile(filePath) + + if err != nil { + return "", err + } + + encoded := base64.StdEncoding.EncodeToString(data) + + return encoded, nil +} + +func (p *Yolo8Plugin) detect(image string) (classify.Labels, error) { + client := &http.Client{Timeout: 60 * time.Second} + url := fmt.Sprintf("http://%s:%s/detect", p.hostname, p.port) + payload := map[string]string{"image": image} + + var req *http.Request + var output *DetectResults + + if j, err := json.Marshal(payload); err != nil { + return nil, err + } else if req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(j)); err != nil { + return nil, err + } + + // Add Content-Type header. + req.Header.Add("Content-Type", "application/json") + + if resp, err := client.Do(req); err != nil { + return nil, err + } else if resp.StatusCode != 200 { + return nil, fmt.Errorf("yolo8 server running at %s:%s, bad status %d\n", p.hostname, p.port, resp.StatusCode) + } else if body, err := io.ReadAll(resp.Body); err != nil { + return nil, err + } else if err := json.Unmarshal(body, &output); err != nil { + return nil, err + } else { + return (*output).toLabels(p.confThreshold), nil + } +} + +func (p *Yolo8Plugin) classify(image string) (classify.Labels, error) { + client := &http.Client{Timeout: 60 * time.Second} + url := fmt.Sprintf("http://%s:%s/classify", p.hostname, p.port) + payload := map[string]string{"image": image} + + var req *http.Request + var output *ClassifyResults + + if j, err := json.Marshal(payload); err != nil { + return nil, err + } else if req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(j)); err != nil { + return nil, err + } + + // Add Content-Type header. + req.Header.Add("Content-Type", "application/json") + + if resp, err := client.Do(req); err != nil { + return nil, err + } else if resp.StatusCode != 200 { + return nil, fmt.Errorf("yolo8 server running at %s:%s, bad status %d\n", p.hostname, p.port, resp.StatusCode) + } else if body, err := io.ReadAll(resp.Body); err != nil { + return nil, err + } else if err := json.Unmarshal(body, &output); err != nil { + return nil, err + } else { + return (*output).toLabels(p.confThreshold), nil + } +} + +func (results DetectResults) toLabels(threshold float64) classify.Labels { + labels := make(classify.Labels, 0) + + for _, result := range results { + if result.Confidence > threshold { + labels = append(labels, classify.Label{ + Name: result.Name, + // It would be nice to be able to denote that the label comes from the yolo8 plugin, + // but PhotoPrism does not really like it when custom sources are used. + // Source: p.Name(), + Source: classify.SrcImage, + Uncertainty: int((1 - result.Confidence) * 100), + Priority: 0, + }) + } + } + + return labels +} + +func (results ClassifyResults) toLabels(threshold float64) classify.Labels { + labels := make(classify.Labels, 0) + + for label, confidence := range results { + if confidence > threshold { + labels = append(labels, classify.Label{ + Name: label, + // It would be nice to be able to denote that the label comes from the yolo8 plugin, + // but PhotoPrism does not really like it when custom sources are used. + // Source: p.Name(), + Source: classify.SrcImage, + Uncertainty: int((1 - confidence) * 100), + Priority: 0, + }) + } + } + + return labels +} + +// Export the plugin. +var Plugin Yolo8Plugin From 4ed7d35c867b4e1e52ef7383172d2b01e6bacfa3 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Fri, 2 Jun 2023 02:39:07 +0300 Subject: [PATCH 10/16] Plugins: Common configuration and http utilities A couple of common patterns around configuration and calling external http services have been identified, so in order to reduce code duplication, a couple of utility interfaces and methods have been introduced. Additionally the go version in go.mod has been bumped to 1.18, which is required for generics. --- go.mod | 2 +- internal/plugin/config.go | 25 +++++++- internal/plugin/interface.go | 7 +++ internal/plugin/utils.go | 69 +++++++++++++++++++++ internal/plugin/yolo8/yolo8.go | 106 ++++++++------------------------- 5 files changed, 125 insertions(+), 84 deletions(-) create mode 100644 internal/plugin/utils.go diff --git a/go.mod b/go.mod index aeb9507913d..0cd088eb894 100644 --- a/go.mod +++ b/go.mod @@ -128,4 +128,4 @@ require ( golang.org/x/arch v0.2.0 // indirect ) -go 1.17 +go 1.18 diff --git a/internal/plugin/config.go b/internal/plugin/config.go index e3307aa49e0..284741ebb13 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -2,8 +2,9 @@ package plugin import ( "fmt" - "strings" "os" + "strconv" + "strings" "github.com/photoprism/photoprism/pkg/list" ) @@ -25,6 +26,28 @@ func (c PluginConfig) Enabled() bool { return false } +// MandatoryStringParameter reads a mandatory string plugin parameter. +func (c PluginConfig) MandatoryStringParameter(name string) (string, error) { + if value, ok := c[name]; !ok { + return "", fmt.Errorf("%s parameter is mandatory", name) + } else { + return value, nil + } +} + +// OptionalFloatParameter reads an optional float64 plugin parameter. +func (c PluginConfig) OptionalFloatParameter(name string, defaultValue float64) (float64, error) { + if value, ok := c[name]; ok { + if fValue, err := strconv.ParseFloat(value, 64); err != nil { + return 0, err + } else { + return fValue, nil + } + } else { + return defaultValue, nil + } +} + func loadConfig(p Plugin) PluginConfig { var config = make(PluginConfig) diff --git a/internal/plugin/interface.go b/internal/plugin/interface.go index 7036f0b971d..a9b60d80687 100644 --- a/internal/plugin/interface.go +++ b/internal/plugin/interface.go @@ -18,6 +18,13 @@ type Plugin interface { OnIndex(*entity.File, *entity.Photo) error } +// HttpPlugin provides an interface for plugins calling external http services. +type HttpPlugin interface { + Plugin + Hostname() string + Port() string +} + // OnIndex calls the [OnIndex] hook method for all enabled plugins. func OnIndex(file *entity.File, photo *entity.Photo) (changed bool) { for _, p := range getPlugins() { diff --git a/internal/plugin/utils.go b/internal/plugin/utils.go new file mode 100644 index 00000000000..a9da4d002cc --- /dev/null +++ b/internal/plugin/utils.go @@ -0,0 +1,69 @@ +package plugin + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "time" + + "github.com/disintegration/imaging" + "github.com/photoprism/photoprism/pkg/fs" +) + +// ReadImageAsBase64 reads an image, rotates it if needed and returns it as base64-encoded string. +func ReadImageAsBase64(filePath string) (string, error) { + if !fs.FileExists(filePath) { + return "", fmt.Errorf("file %s is missing", filepath.Base(filePath)) + } + + img, err := imaging.Open(filePath, imaging.AutoOrientation(true)) + if err != nil { + return "", err + } + + buffer := &bytes.Buffer{} + err = imaging.Encode(buffer, img, imaging.JPEG) + if err != nil { + return "", err + } + + encoded := base64.StdEncoding.EncodeToString(buffer.Bytes()) + + return encoded, nil +} + +// PostJson sends a post request with a json payload to a plugin endpoint and returns a deserialized json output. +func PostJson[T any](p HttpPlugin, endpoint string, payload map[string]interface{}) (T, error) { + client := &http.Client{Timeout: 60 * time.Second} + url := fmt.Sprintf("http://%s:%s/%s", p.Hostname(), p.Port(), endpoint) + + var empty T + + var req *http.Request + var output *T + + if j, err := json.Marshal(payload); err != nil { + return empty, err + } else if req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(j)); err != nil { + return empty, err + } + + // Add Content-Type header. + req.Header.Add("Content-Type", "application/json") + + if resp, err := client.Do(req); err != nil { + return empty, err + } else if resp.StatusCode != 200 { + return empty, fmt.Errorf("%s server running at %s:%s, bad status %d", p.Name(), p.Hostname(), p.Port(), resp.StatusCode) + } else if body, err := io.ReadAll(resp.Body); err != nil { + return empty, err + } else if err := json.Unmarshal(body, &output); err != nil { + return empty, err + } else { + return *output, nil + } +} diff --git a/internal/plugin/yolo8/yolo8.go b/internal/plugin/yolo8/yolo8.go index c6c3ca3795d..e543406fa4f 100644 --- a/internal/plugin/yolo8/yolo8.go +++ b/internal/plugin/yolo8/yolo8.go @@ -1,22 +1,12 @@ package main import ( - "bytes" - "encoding/base64" - "encoding/json" "fmt" - "io" - "io/ioutil" - "net/http" - "strconv" - "time" "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/photoprism" "github.com/photoprism/photoprism/internal/plugin" - "github.com/photoprism/photoprism/pkg/clean" - "github.com/photoprism/photoprism/pkg/fs" ) type ClassifyResults map[string]float64 @@ -38,24 +28,28 @@ func (p Yolo8Plugin) Name() string { return "yolo8" } +func (p Yolo8Plugin) Hostname() string { + return p.hostname +} + +func (p Yolo8Plugin) Port() string { + return p.port +} + func (p *Yolo8Plugin) Configure(config plugin.PluginConfig) error { - hostname, ok := config["hostname"] - if !ok { - return fmt.Errorf("hostname parameter is mandatory") + hostname, err := config.MandatoryStringParameter("hostname") + if err != nil { + return err } - port, ok := config["port"] - if !ok { - return fmt.Errorf("port parameter is mandatory") + port, err := config.MandatoryStringParameter("port") + if err != nil { + return err } - threshold := 0.5 - var err error - - if t, ok := config["confidence_threshold"]; ok { - if threshold, err = strconv.ParseFloat(t, 64); err != nil { - return err - } + threshold, err := config.OptionalFloatParameter("confidence_threshold", 0.5) + if err != nil { + return err } p.hostname = hostname @@ -85,78 +79,26 @@ func (p *Yolo8Plugin) OnIndex(file *entity.File, photo *entity.Photo) error { func (p *Yolo8Plugin) image(f *entity.File) (string, error) { filePath := photoprism.FileName(f.FileRoot, f.FileName) - if !fs.FileExists(filePath) { - return "", fmt.Errorf("file %s is missing", clean.Log(f.FileName)) - } - - data, err := ioutil.ReadFile(filePath) - - if err != nil { - return "", err - } - - encoded := base64.StdEncoding.EncodeToString(data) - - return encoded, nil + return plugin.ReadImageAsBase64(filePath) } func (p *Yolo8Plugin) detect(image string) (classify.Labels, error) { - client := &http.Client{Timeout: 60 * time.Second} - url := fmt.Sprintf("http://%s:%s/detect", p.hostname, p.port) - payload := map[string]string{"image": image} - - var req *http.Request - var output *DetectResults + payload := map[string]interface{}{"image": image} - if j, err := json.Marshal(payload); err != nil { - return nil, err - } else if req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(j)); err != nil { - return nil, err - } - - // Add Content-Type header. - req.Header.Add("Content-Type", "application/json") - - if resp, err := client.Do(req); err != nil { - return nil, err - } else if resp.StatusCode != 200 { - return nil, fmt.Errorf("yolo8 server running at %s:%s, bad status %d\n", p.hostname, p.port, resp.StatusCode) - } else if body, err := io.ReadAll(resp.Body); err != nil { - return nil, err - } else if err := json.Unmarshal(body, &output); err != nil { + if output, err := plugin.PostJson[DetectResults](p, "detect", payload); err != nil { return nil, err } else { - return (*output).toLabels(p.confThreshold), nil + return output.toLabels(p.confThreshold), nil } } func (p *Yolo8Plugin) classify(image string) (classify.Labels, error) { - client := &http.Client{Timeout: 60 * time.Second} - url := fmt.Sprintf("http://%s:%s/classify", p.hostname, p.port) - payload := map[string]string{"image": image} + payload := map[string]interface{}{"image": image} - var req *http.Request - var output *ClassifyResults - - if j, err := json.Marshal(payload); err != nil { - return nil, err - } else if req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(j)); err != nil { - return nil, err - } - - // Add Content-Type header. - req.Header.Add("Content-Type", "application/json") - - if resp, err := client.Do(req); err != nil { - return nil, err - } else if resp.StatusCode != 200 { - return nil, fmt.Errorf("yolo8 server running at %s:%s, bad status %d\n", p.hostname, p.port, resp.StatusCode) - } else if body, err := io.ReadAll(resp.Body); err != nil { - return nil, err - } else if err := json.Unmarshal(body, &output); err != nil { + if output, err := plugin.PostJson[ClassifyResults](p, "classify", payload); err != nil { return nil, err } else { - return (*output).toLabels(p.confThreshold), nil + return output.toLabels(p.confThreshold), nil } } From 1029890f368b88e92b55f16655aa11c992548f60 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sat, 3 Jun 2023 01:46:29 +0300 Subject: [PATCH 11/16] Plugin: Upscaling plugin based on Real ESRGAN --- Makefile | 5 +- docker-compose.plugins.yml | 4 + ext/realesrgan/main.py | 3 - internal/plugin/realesrgan/realesrgan.go | 139 +++++++++++++++++++++++ 4 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 internal/plugin/realesrgan/realesrgan.go diff --git a/Makefile b/Makefile index 596e6ca610b..7692eac0f2e 100644 --- a/Makefile +++ b/Makefile @@ -224,10 +224,13 @@ build-race: build-static: rm -f $(BINARY_NAME) scripts/build.sh static $(BINARY_NAME) -build-plugins: build-plugin-demo build-plugin-yolo8 +build-plugins: build-plugin-demo build-plugin-realesrgan build-plugin-yolo8 build-plugin-demo: mkdir -p $(PLUGINS_PATH) go build -tags=debug -buildmode=plugin -o $(PLUGINS_PATH)/demo.so internal/plugin/demo/demo.go +build-plugin-realesrgan: + mkdir -p $(PLUGINS_PATH) + go build -tags=debug -buildmode=plugin -o $(PLUGINS_PATH)/realesrgan.so internal/plugin/realesrgan/realesrgan.go build-plugin-yolo8: mkdir -p $(PLUGINS_PATH) go build -tags=debug -buildmode=plugin -o $(PLUGINS_PATH)/yolo8.so internal/plugin/yolo8/yolo8.go diff --git a/docker-compose.plugins.yml b/docker-compose.plugins.yml index 664931b1e4b..1e6ba85f096 100644 --- a/docker-compose.plugins.yml +++ b/docker-compose.plugins.yml @@ -10,6 +10,9 @@ services: PHOTOPRISM_PLUGIN_YOLO8_ENABLED: "true" PHOTOPRISM_PLUGIN_YOLO8_HOSTNAME: "yolo8" PHOTOPRISM_PLUGIN_YOLO8_PORT: "5000" + PHOTOPRISM_PLUGIN_REALESRGAN_ENABLED: "true" + PHOTOPRISM_PLUGIN_REALESRGAN_HOSTNAME: "realesrgan" + PHOTOPRISM_PLUGIN_REALESRGAN_PORT: "5001" ## Image classification API yolo8: @@ -20,3 +23,4 @@ services: realesrgan: image: kvalev/realesrgan:0.3.0 pull_policy: always + restart: always diff --git a/ext/realesrgan/main.py b/ext/realesrgan/main.py index 3d026e0776b..9ffe641b7e0 100644 --- a/ext/realesrgan/main.py +++ b/ext/realesrgan/main.py @@ -63,9 +63,6 @@ model_path=model_path, model=model_info["model"](), half=True if "cuda" in device else False, - tile=512, - tile_pad=10, - pre_pad=10, device=device, ) diff --git a/internal/plugin/realesrgan/realesrgan.go b/internal/plugin/realesrgan/realesrgan.go new file mode 100644 index 00000000000..3125a28ef64 --- /dev/null +++ b/internal/plugin/realesrgan/realesrgan.go @@ -0,0 +1,139 @@ +package main + +import ( + "encoding/base64" + "os" + + "github.com/photoprism/photoprism/internal/entity" + "github.com/photoprism/photoprism/internal/get" + "github.com/photoprism/photoprism/internal/photoprism" + "github.com/photoprism/photoprism/internal/plugin" + "github.com/photoprism/photoprism/pkg/fs" +) + +type UpscaleResult struct { + Image string `json:"image"` +} + +type RealESRGANPlugin struct { + hostname string + port string + scale float64 + + thresholdResolution float64 + thresholdQuality float64 +} + +func (p RealESRGANPlugin) Name() string { + return "realesrgan" +} + +func (p RealESRGANPlugin) Hostname() string { + return p.hostname +} + +func (p RealESRGANPlugin) Port() string { + return p.port +} + +func (p *RealESRGANPlugin) Configure(config plugin.PluginConfig) error { + hostname, err := config.MandatoryStringParameter("hostname") + if err != nil { + return err + } + + port, err := config.MandatoryStringParameter("port") + if err != nil { + return err + } + + scale, err := config.OptionalFloatParameter("scale", 2) + if err != nil { + return err + } + + resolution, err := config.OptionalFloatParameter("threshold_resolution", 3) + if err != nil { + return err + } + + quality, err := config.OptionalFloatParameter("threshold_quality", 3) + if err != nil { + return err + } + + p.hostname = hostname + p.port = port + p.scale = scale + p.thresholdResolution = resolution + p.thresholdQuality = quality + + return nil +} + +func (p *RealESRGANPlugin) OnIndex(file *entity.File, photo *entity.Photo) error { + if !p.needsUpscaling(photo) { + return nil + } + + image, err := p.image(file) + if err != nil { + return err + } + + output, err := p.superscale(image) + if err != nil { + return err + } + + if err := p.save(file, output); err != nil { + return err + } + + return nil +} + +func (p *RealESRGANPlugin) needsUpscaling(photo *entity.Photo) bool { + return p.thresholdResolution > float64(photo.PhotoResolution) || p.thresholdQuality > float64(photo.PhotoQuality) +} + +func (p *RealESRGANPlugin) image(f *entity.File) (string, error) { + filePath := photoprism.FileName(f.FileRoot, f.FileName) + + return plugin.ReadImageAsBase64(filePath) +} + +func (p *RealESRGANPlugin) superscale(image string) ([]byte, error) { + payload := map[string]interface{}{"image": image, "scale": p.scale} + + if output, err := plugin.PostJson[UpscaleResult](p, "superscale", payload); err != nil { + return nil, err + } else { + if decoded, err := base64.StdEncoding.DecodeString(output.Image); err != nil { + return nil, err + } else { + return decoded, nil + } + } +} + +func (p *RealESRGANPlugin) save(f *entity.File, data []byte) error { + conf := get.Config() + + ext := ".SUPERSCALED" + fs.ExtJPEG + baseDir := conf.OriginalsPath() + // if f.InSidecar() { + // baseDir = conf.SidecarPath() + // } + + imageName := fs.FileName(photoprism.FileName(f.FileRoot, f.FileName), conf.SidecarPath(), baseDir, ext) + + if err := os.WriteFile(imageName, data, 0666); err != nil { + return err + } + + return nil +} + +// Export the plugin. +var Plugin RealESRGANPlugin From 29e9952a9cf9b2455265e57c3397ded68eb895bc Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sat, 3 Jun 2023 02:23:33 +0300 Subject: [PATCH 12/16] Plugin: Add YOLO8 models to gitignore --- ext/yolo8/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 ext/yolo8/.gitignore diff --git a/ext/yolo8/.gitignore b/ext/yolo8/.gitignore new file mode 100644 index 00000000000..4b6ebe5ff71 --- /dev/null +++ b/ext/yolo8/.gitignore @@ -0,0 +1 @@ +*.pt From 7534aaa04bd216901624d06dd43fe9d87d527294 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sun, 4 Jun 2023 01:58:37 +0300 Subject: [PATCH 13/16] Plugins: Common way of handling plugin configuration --- go.mod | 2 + go.sum | 4 ++ internal/plugin/config.go | 30 ++++++-------- internal/plugin/realesrgan/realesrgan.go | 53 +++++++----------------- internal/plugin/yolo8/yolo8.go | 40 ++++++++---------- 5 files changed, 51 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 0cd088eb894..bbf14e968b9 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require github.com/go-ldap/ldap/v3 v3.4.5-0.20230210083308-d16fb563008d require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 // indirect @@ -100,6 +101,7 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/mandykoh/go-parallel v0.1.0 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index a893ccf6a84..aaaf77c558c 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -305,6 +307,8 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/internal/plugin/config.go b/internal/plugin/config.go index 284741ebb13..eb362afe4da 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -3,9 +3,10 @@ package plugin import ( "fmt" "os" - "strconv" "strings" + "github.com/creasty/defaults" + "github.com/mitchellh/mapstructure" "github.com/photoprism/photoprism/pkg/list" ) @@ -26,26 +27,19 @@ func (c PluginConfig) Enabled() bool { return false } -// MandatoryStringParameter reads a mandatory string plugin parameter. -func (c PluginConfig) MandatoryStringParameter(name string) (string, error) { - if value, ok := c[name]; !ok { - return "", fmt.Errorf("%s parameter is mandatory", name) - } else { - return value, nil +// // Decode populates a struct with the configuration data. +func (c PluginConfig) Decode(init any) error { + if err := defaults.Set(init); err != nil { + return err } -} -// OptionalFloatParameter reads an optional float64 plugin parameter. -func (c PluginConfig) OptionalFloatParameter(name string, defaultValue float64) (float64, error) { - if value, ok := c[name]; ok { - if fValue, err := strconv.ParseFloat(value, 64); err != nil { - return 0, err - } else { - return fValue, nil - } - } else { - return defaultValue, nil + if err := mapstructure.WeakDecode(c, &init); err != nil { + return err } + + fmt.Printf("CFG OK %#v", init) + + return nil } func loadConfig(p Plugin) PluginConfig { diff --git a/internal/plugin/realesrgan/realesrgan.go b/internal/plugin/realesrgan/realesrgan.go index 3125a28ef64..b591fab56ad 100644 --- a/internal/plugin/realesrgan/realesrgan.go +++ b/internal/plugin/realesrgan/realesrgan.go @@ -15,13 +15,17 @@ type UpscaleResult struct { Image string `json:"image"` } -type RealESRGANPlugin struct { - hostname string - port string - scale float64 +type Config struct { + Hostname string + Port string + Scale float64 `default:"2"` + + ThresholdResolution float64 `default:"3"` + ThresholdQuality float64 `default:"3"` +} - thresholdResolution float64 - thresholdQuality float64 +type RealESRGANPlugin struct { + Config *Config } func (p RealESRGANPlugin) Name() string { @@ -29,45 +33,20 @@ func (p RealESRGANPlugin) Name() string { } func (p RealESRGANPlugin) Hostname() string { - return p.hostname + return p.Config.Hostname } func (p RealESRGANPlugin) Port() string { - return p.port + return p.Config.Port } func (p *RealESRGANPlugin) Configure(config plugin.PluginConfig) error { - hostname, err := config.MandatoryStringParameter("hostname") - if err != nil { - return err - } + p.Config = &Config{} - port, err := config.MandatoryStringParameter("port") - if err != nil { + if err := config.Decode(p.Config); err != nil { return err } - scale, err := config.OptionalFloatParameter("scale", 2) - if err != nil { - return err - } - - resolution, err := config.OptionalFloatParameter("threshold_resolution", 3) - if err != nil { - return err - } - - quality, err := config.OptionalFloatParameter("threshold_quality", 3) - if err != nil { - return err - } - - p.hostname = hostname - p.port = port - p.scale = scale - p.thresholdResolution = resolution - p.thresholdQuality = quality - return nil } @@ -94,7 +73,7 @@ func (p *RealESRGANPlugin) OnIndex(file *entity.File, photo *entity.Photo) error } func (p *RealESRGANPlugin) needsUpscaling(photo *entity.Photo) bool { - return p.thresholdResolution > float64(photo.PhotoResolution) || p.thresholdQuality > float64(photo.PhotoQuality) + return p.Config.ThresholdResolution > float64(photo.PhotoResolution) || p.Config.ThresholdQuality > float64(photo.PhotoQuality) } func (p *RealESRGANPlugin) image(f *entity.File) (string, error) { @@ -104,7 +83,7 @@ func (p *RealESRGANPlugin) image(f *entity.File) (string, error) { } func (p *RealESRGANPlugin) superscale(image string) ([]byte, error) { - payload := map[string]interface{}{"image": image, "scale": p.scale} + payload := map[string]interface{}{"image": image, "scale": p.Config.Scale} if output, err := plugin.PostJson[UpscaleResult](p, "superscale", payload); err != nil { return nil, err diff --git a/internal/plugin/yolo8/yolo8.go b/internal/plugin/yolo8/yolo8.go index e543406fa4f..951bf88081f 100644 --- a/internal/plugin/yolo8/yolo8.go +++ b/internal/plugin/yolo8/yolo8.go @@ -18,10 +18,14 @@ type DetectResult struct { Confidence float64 `json:"confidence"` } +type Config struct { + Hostname string + Port string + ThresholdConfidence float64 `default:"0.4"` +} + type Yolo8Plugin struct { - hostname string - port string - confThreshold float64 + Config *Config } func (p Yolo8Plugin) Name() string { @@ -29,33 +33,20 @@ func (p Yolo8Plugin) Name() string { } func (p Yolo8Plugin) Hostname() string { - return p.hostname + return p.Config.Hostname } func (p Yolo8Plugin) Port() string { - return p.port + return p.Config.Port } func (p *Yolo8Plugin) Configure(config plugin.PluginConfig) error { - hostname, err := config.MandatoryStringParameter("hostname") - if err != nil { - return err - } + p.Config = &Config{} - port, err := config.MandatoryStringParameter("port") - if err != nil { + if err := config.Decode(p.Config); err != nil { return err } - threshold, err := config.OptionalFloatParameter("confidence_threshold", 0.5) - if err != nil { - return err - } - - p.hostname = hostname - p.port = port - p.confThreshold = threshold - return nil } @@ -70,8 +61,11 @@ func (p *Yolo8Plugin) OnIndex(file *entity.File, photo *entity.Photo) error { return err } + if len(labels) > 0 { + fmt.Printf("adding labels %#v", labels) + } + photo.AddLabels(labels) - fmt.Printf("adding labels %#v", labels) return nil } @@ -88,7 +82,7 @@ func (p *Yolo8Plugin) detect(image string) (classify.Labels, error) { if output, err := plugin.PostJson[DetectResults](p, "detect", payload); err != nil { return nil, err } else { - return output.toLabels(p.confThreshold), nil + return output.toLabels(p.Config.ThresholdConfidence), nil } } @@ -98,7 +92,7 @@ func (p *Yolo8Plugin) classify(image string) (classify.Labels, error) { if output, err := plugin.PostJson[ClassifyResults](p, "classify", payload); err != nil { return nil, err } else { - return output.toLabels(p.confThreshold), nil + return output.toLabels(p.Config.ThresholdConfidence), nil } } From e63892b9263a26f1fc8289cf8936715485d33298 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sun, 4 Jun 2023 02:51:38 +0300 Subject: [PATCH 14/16] Plugins: Interface for plugin logging --- internal/plugin/config.go | 2 -- internal/plugin/logging.go | 25 ++++++++++++++++++++++++ internal/plugin/realesrgan/realesrgan.go | 2 ++ internal/plugin/yolo8/yolo8.go | 6 +++--- 4 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 internal/plugin/logging.go diff --git a/internal/plugin/config.go b/internal/plugin/config.go index eb362afe4da..7335330d259 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -37,8 +37,6 @@ func (c PluginConfig) Decode(init any) error { return err } - fmt.Printf("CFG OK %#v", init) - return nil } diff --git a/internal/plugin/logging.go b/internal/plugin/logging.go new file mode 100644 index 00000000000..0954fa63168 --- /dev/null +++ b/internal/plugin/logging.go @@ -0,0 +1,25 @@ +package plugin + +import ( + "fmt" + "strings" +) + +func logPrefix(p Plugin) string { + return fmt.Sprintf("plugin %s: ", strings.ToLower(p.Name())) +} + +// LogDebugf logs a debug message with the common log prefix. +func LogDebugf(p Plugin, format string, args ...any) { + log.Debugf(logPrefix(p)+format, args...) +} + +// LogInfof logs an info message with the common log prefix. +func LogInfof(p Plugin, format string, args ...any) { + log.Infof(logPrefix(p)+format, args...) +} + +// LogWarnf logs a warning message with the common log prefix. +func LogWarnf(p Plugin, format string, args ...any) { + log.Warnf(logPrefix(p)+format, args...) +} diff --git a/internal/plugin/realesrgan/realesrgan.go b/internal/plugin/realesrgan/realesrgan.go index b591fab56ad..92d0cae670f 100644 --- a/internal/plugin/realesrgan/realesrgan.go +++ b/internal/plugin/realesrgan/realesrgan.go @@ -47,6 +47,8 @@ func (p *RealESRGANPlugin) Configure(config plugin.PluginConfig) error { return err } + plugin.LogDebugf(p, "configuration loaded %#v", p.Config) + return nil } diff --git a/internal/plugin/yolo8/yolo8.go b/internal/plugin/yolo8/yolo8.go index 951bf88081f..0041eef7832 100644 --- a/internal/plugin/yolo8/yolo8.go +++ b/internal/plugin/yolo8/yolo8.go @@ -1,8 +1,6 @@ package main import ( - "fmt" - "github.com/photoprism/photoprism/internal/classify" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/photoprism" @@ -47,6 +45,8 @@ func (p *Yolo8Plugin) Configure(config plugin.PluginConfig) error { return err } + plugin.LogDebugf(p, "configuration loaded %#v", p.Config) + return nil } @@ -62,7 +62,7 @@ func (p *Yolo8Plugin) OnIndex(file *entity.File, photo *entity.Photo) error { } if len(labels) > 0 { - fmt.Printf("adding labels %#v", labels) + plugin.LogDebugf(p, "adding labels %#v", labels) } photo.AddLabels(labels) From c9efe73c6cbec5b5f341e282637ac8db54a8dc99 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sun, 4 Jun 2023 02:59:33 +0300 Subject: [PATCH 15/16] Plugins: CI job that builds and archives plugin solibs --- .../workflows/plugins-build-and-archive.yaml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/plugins-build-and-archive.yaml diff --git a/.github/workflows/plugins-build-and-archive.yaml b/.github/workflows/plugins-build-and-archive.yaml new file mode 100644 index 00000000000..7d043c60c1f --- /dev/null +++ b/.github/workflows/plugins-build-and-archive.yaml @@ -0,0 +1,42 @@ +name: Build and archive plugin solibs + +on: + push: + branches: [ develop, preview ] + pull_request: + branches: [ develop, preview ] + + workflow_dispatch: + +jobs: + build-and-archive-plugins: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Create compose stack + run: docker-compose -f docker-compose.ci.yml up -d --build --force-recreate + + - name: Compile plugins + id: compile_plugins + run: docker-compose -f docker-compose.ci.yml exec -T photoprism make build-plugins + + - name: Look for plugin solibs + run: docker-compose -f docker-compose.ci.yml exec -T photoprism find storage/plugins/ + if: always() && steps.compile_plugins.outcome == 'success' + + - name: Copy plugins from container to host + run: docker compose -f docker-compose.ci.yml cp photoprism:/go/src/github.com/photoprism/photoprism/storage/plugins/ plugins/ + if: always() && steps.compile_plugins.outcome == 'success' + + - name: Tear down stack + run: docker-compose -f docker-compose.ci.yml down + + - name: Archive plugins + uses: actions/upload-artifact@v3 + if: always() && steps.compile_plugins.outcome == 'success' + continue-on-error: true + with: + name: plugins + path: plugins/ From bf35963a8d15f5a74e3f7d69c37e5e9d0e65a090 Mon Sep 17 00:00:00 2001 From: Krassimir Valev Date: Sun, 4 Jun 2023 03:14:48 +0300 Subject: [PATCH 16/16] CI: Pin the GitHub bake action version --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6a481280480..020e10321ac 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -33,7 +33,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - uses: docker/bake-action@master + uses: docker/bake-action@v3 with: files: ./docker/docker-bake.hcl push: true