fossar · jtojnar · Sep 14, 2020 · May 4, 2020 · May 4, 2020 · May 4, 2020
diff --git a/NEWS.md b/NEWS.md
@@ -57,6 +57,7 @@
   ```
 
   ([#1017](https://github.com/SSilence/selfoss/pull/1017), [#1035](https://github.com/SSilence/selfoss/pull/1035))
+- Spouts can now implement `getSourceIcon()` instead of `getIcon()` when icon is associated with the feed, not individual icons. ([#1190](https://github.com/SSilence/selfoss/pull/1190))
 - Some language files have been renamed to use correct [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) and you might need to change the `language` key in your `config.ini`:
   * Simplified Chinese `zh-CN`
   * Traditional Chinese `zh-TW`

diff --git a/src/common.php b/src/common.php
@@ -157,6 +157,32 @@
     'constructParams' => $dbParams
 ]));
 
+$dice->addRule('$iconStorageBackend', [
+    'instanceOf' => helpers\Storage\FileStorage::class,
+    'constructParams' => [
+        \F3::get('datadir') . '/favicons'
+    ],
+]);
+
+$dice->addRule(helpers\IconStore::class, array_merge($shared, [
+    'constructParams' => [
+        ['instance' => '$iconStorageBackend'],
+    ]
+]));
+
+$dice->addRule('$thumbnailStorageBackend', [
+    'instanceOf' => helpers\Storage\FileStorage::class,
+    'constructParams' => [
+        \F3::get('datadir') . '/thumbnails'
+    ],
+]);
+
+$dice->addRule(helpers\ThumbnailStore::class, array_merge($shared, [
+    'constructParams' => [
+        ['instance' => '$thumbnailStorageBackend'],
+    ]
+]));
+
 // Fallback rule
 $dice->addRule('*', $substitutions);
 

diff --git a/src/helpers/ContentLoader.php b/src/helpers/ContentLoader.php
@@ -15,6 +15,9 @@ class ContentLoader {
     /** @var \daos\Database database for optimization */
     private $database;
 
+    /** @var IconStore icon store */
+    private $iconStore;
+
     /** @var Image image helper */
     private $imageHelper;
 
@@ -30,16 +33,28 @@ class ContentLoader {
     /** @var SpoutLoader spout loader */
     private $spoutLoader;
 
+    /** @var ThumbnailStore thumbnail store */
+    private $thumbnailStore;
+
+    /** @var WebClient thumbnail store */
+    private $webClient;
+
+    const ICON_FORMAT = Image::FORMAT_PNG;
+    const THUMBNAIL_FORMAT = Image::FORMAT_JPEG;
+
     /**
      * ctor
      */
-    public function __construct(\daos\Database $database, Image $imageHelper, \daos\Items $itemsDao, Logger $logger, \daos\Sources $sourcesDao, SpoutLoader $spoutLoader) {
+    public function __construct(\daos\Database $database, IconStore $iconStore, Image $imageHelper, \daos\Items $itemsDao, Logger $logger, \daos\Sources $sourcesDao, SpoutLoader $spoutLoader, ThumbnailStore $thumbnailStore, WebClient $webClient) {
         $this->database = $database;
+        $this->iconStore = $iconStore;
         $this->imageHelper = $imageHelper;
         $this->itemsDao = $itemsDao;
         $this->logger = $logger;
         $this->sourcesDao = $sourcesDao;
         $this->spoutLoader = $spoutLoader;
+        $this->thumbnailStore = $thumbnailStore;
+        $this->webClient = $webClient;
     }
 
     /**
@@ -134,7 +149,8 @@ public function fetch($source) {
         }
         $itemsFound = $this->itemsDao->findAll($itemsInFeed, $source['id']);
 
-        $lasticon = null;
+        $iconCache = [];
+        $sourceIconUrl = null;
         $itemsSeen = [];
         foreach ($spout as $item) {
             // item already in database?
@@ -192,31 +208,58 @@ public function fetch($source) {
 
             $this->logger->debug('item content sanitized');
 
-            try {
-                $icon = $item->getIcon();
-            } catch (\Exception $e) {
-                $this->logger->debug('icon: error', ['exception' => $e]);
-
-                return;
-            }
-
             $newItem = [
                 'title' => $title,
                 'content' => $content,
                 'source' => $source['id'],
                 'datetime' => $itemDate->format('Y-m-d H:i:s'),
                 'uid' => $item->getId(),
-                'thumbnail' => $item->getThumbnail(),
-                'icon' => $icon !== null ? $icon : '',
                 'link' => htmLawed($item->getLink(), ['deny_attribute' => '*', 'elements' => '-*']),
                 'author' => $author
             ];
 
-            // save thumbnail
-            $newItem = $this->fetchThumbnail($item->getThumbnail(), $newItem);
+            $thumbnailUrl = $item->getThumbnail();
+            if (strlen(trim($thumbnailUrl)) > 0) {
+                // save thumbnail
+                $newItem['thumbnail'] = $this->fetchThumbnail($thumbnailUrl) ?: '';
+            }
 
-            // save icon
-            $newItem = $this->fetchIcon($item->getIcon(), $newItem, $lasticon);
+            try {
+                $iconUrl = $item->getIcon();
+                if (strlen(trim($iconUrl)) > 0) {
+                    if (isset($iconCache[$iconUrl])) {
+                        $this->logger->debug('reusing recently used icon: ' . $iconUrl);
+                    } else {
+                        // save icon
+                        $iconCache[$iconUrl] = $this->fetchIcon($iconUrl) ?: '';
+                    }
+                    $newItem['icon'] = $iconCache[$iconUrl];
+                } elseif ($sourceIconUrl !== null) {
+                    $this->logger->debug('using the source icon');
+                    $newItem['icon'] = $sourceIconUrl;
+                } else {
+                    try {
+                        // we do not want to run this more than once
+                        $sourceIconUrl = $item->getSourceIcon() ?: '';
+
+                        if (strlen(trim($sourceIconUrl)) > 0) {
+                            // save source icon
+                            $sourceIconUrl = $this->fetchIcon($sourceIconUrl) ?: '';
+                            $newItem['icon'] = $sourceIconUrl;
+                        } else {
+                            $this->logger->debug('no icon for this item or source');
+                        }
+                    } catch (\Exception $e) {
+                        // cache failure
+                        $sourceIconUrl = '';
+                        $this->logger->error('feed icon: error', ['exception' => $e]);
+                    }
+                }
+            } catch (\Exception $e) {
+                // cache failure
+                $iconCache[$iconUrl] = '';
+                $this->logger->error('icon: error', ['exception' => $e]);
+            }
 
             // insert new item
             $this->itemsDao->add($newItem);
@@ -308,76 +351,57 @@ protected function sanitizeField($value) {
     }
 
     /**
-     * Fetch the thumbanil of a given item
+     * Fetch an image URL and process it as a thumbnail.
      *
-     * @param string $thumbnail the thumbnail url
-     * @param array $newItem new item for saving in database
+     * @param string $url the thumbnail URL
      *
-     * @return array the newItem Object with thumbnail
+     * @return ?string path in the thumbnails directory
      */
-    protected function fetchThumbnail($thumbnail, array $newItem) {
-        if (strlen(trim($thumbnail)) > 0) {
-            $extension = 'jpg';
-            $thumbnailAsJpg = $this->imageHelper->loadImage($thumbnail, $extension, 500, 500);
-            if ($thumbnailAsJpg !== null) {
-                $written = file_put_contents(
-                    \F3::get('datadir') . '/thumbnails/' . md5($thumbnail) . '.' . $extension,
-                    $thumbnailAsJpg
-                );
-                if ($written !== false) {
-                    $newItem['thumbnail'] = md5($thumbnail) . '.' . $extension;
-                    $this->logger->debug('Thumbnail generated: ' . $thumbnail);
-                } else {
-                    $this->logger->warning('Unable to store thumbnail: ' . $thumbnail . '. Please check permissions of ' . \F3::get('datadir') . '/thumbnails.');
-                }
+    protected function fetchThumbnail($url) {
+        try {
+            $data = $this->webClient->request($url);
+            $format = self::THUMBNAIL_FORMAT;
+            $image = $this->imageHelper->loadImage($data, $format, 500, 500);
+
+            if ($image !== null) {
+                return $this->thumbnailStore->store($url, $image->getData());
             } else {
-                $newItem['thumbnail'] = '';
-                $this->logger->error('thumbnail generation error: ' . $thumbnail);
+                $this->logger->error('thumbnail generation error: ' . $url);
             }
+        } catch (\Exception $e) {
+            $this->logger->error("failed to retrieve thumbnail $url,", ['exception' => $e]);
+
+            return null;
         }
 
-        return $newItem;
+        return null;
     }
 
     /**
-     * Fetch the icon of a given feed item
+     * Fetch an image and process it as favicon.
      *
-     * @param string $icon icon given by the spout
-     * @param array $newItem new item for saving in database
-     * @param &string $lasticon the last fetched icon
+     * @param string $url icon given by the spout
      *
-     * @return mixed newItem with icon
+     * @return ?string path in the favicons directory
      */
-    protected function fetchIcon($icon, array $newItem, &$lasticon) {
-        if (strlen(trim($icon)) > 0) {
-            $extension = 'png';
-            if ($icon === $lasticon) {
-                $this->logger->debug('use last icon: ' . $lasticon);
-                $newItem['icon'] = md5($lasticon) . '.' . $extension;
+    protected function fetchIcon($url) {
+        try {
+            $data = $this->webClient->request($url);
+            $format = Image::FORMAT_PNG;
+            $image = $this->imageHelper->loadImage($data, $format, 30, null);
+
+            if ($image !== null) {
+                return $this->iconStore->store($url, $image->getData());
             } else {
-                $iconAsPng = $this->imageHelper->loadImage($icon, $extension, 30, null);
-                if ($iconAsPng !== null) {
-                    $written = file_put_contents(
-                        \F3::get('datadir') . '/favicons/' . md5($icon) . '.' . $extension,
-                        $iconAsPng
-                    );
-                    $lasticon = $icon;
-                    if ($written !== false) {
-                        $newItem['icon'] = md5($icon) . '.' . $extension;
-                        $this->logger->debug('Icon generated: ' . $icon);
-                    } else {
-                        $this->logger->warning('Unable to store icon: ' . $icon . '. Please check permissions of ' . \F3::get('datadir') . '/favicons.');
-                    }
-                } else {
-                    $newItem['icon'] = '';
-                    $this->logger->error('icon generation error: ' . $icon);
-                }
+                $this->logger->error('icon generation error: ' . $url);
             }
-        } else {
-            $this->logger->debug('no icon for this feed');
+        } catch (\Exception $e) {
+            $this->logger->error("failed to retrieve image $url,", ['exception' => $e]);
+
+            return null;
         }
 
-        return $newItem;
+        return null;
     }
 
     /**
@@ -428,12 +452,16 @@ public function cleanup() {
 
         // delete orphaned thumbnails
         $this->logger->debug('delete orphaned thumbnails');
-        $this->cleanupFiles('thumbnails');
+        $this->thumbnailStore->cleanup(function($file) {
+            return $this->itemsDao->hasThumbnail($file);
+        });
         $this->logger->debug('delete orphaned thumbnails finished');
 
         // delete orphaned icons
         $this->logger->debug('delete orphaned icons');
-        $this->cleanupFiles('icons');
+        $this->iconStore->cleanup(function($file) {
+            return $this->itemsDao->hasIcon($file);
+        });
         $this->logger->debug('delete orphaned icons finished');
 
         // optimize database
@@ -442,36 +470,6 @@ public function cleanup() {
         $this->logger->debug('optimize database finished');
     }
 
-    /**
-     * clean up orphaned thumbnails or icons
-     *
-     * @param string $type thumbnails or icons
-     *
-     * @return void
-     */
-    protected function cleanupFiles($type) {
-        if ($type === 'thumbnails') {
-            $checker = function($file) {
-                return $this->itemsDao->hasThumbnail($file);
-            };
-            $itemPath = \F3::get('datadir') . '/thumbnails/';
-        } elseif ($type === 'icons') {
-            $checker = function($file) {
-                return $this->itemsDao->hasIcon($file);
-            };
-            $itemPath = \F3::get('datadir') . '/favicons/';
-        }
-
-        foreach (scandir($itemPath) as $file) {
-            if (is_file($itemPath . $file) && $file !== '.htaccess') {
-                $inUsage = $checker($file);
-                if ($inUsage === false) {
-                    unlink($itemPath . $file);
-                }
-            }
-        }
-    }
-
     /**
      * Update source (remove previous errors, update last update)
      *

diff --git a/src/helpers/IconStore.php b/src/helpers/IconStore.php
@@ -0,0 +1,48 @@
+<?php
+
+namespace helpers;
+
+use helpers\Storage\FileStorage;
+use Monolog\Logger;
+
+/**
+ * Icon storage.
+ */
+class IconStore {
+    /** @var Logger */
+    private $logger;
+
+    /** @var FileStorage */
+    private $storage;
+
+    public function __construct(FileStorage $storage, Logger $logger) {
+        $this->storage = $storage;
+        $this->logger = $logger;
+    }
+
+    /**
+     * Store given blob as URL.
+     *
+     * @param string $url
+     * @param string $blob
+     *
+     * @return ?string
+     */
+    public function store($url, $blob) {
+        $extension = Image::getExtension(ContentLoader::ICON_FORMAT);
+        $this->logger->debug('Storing icon: ' . $url);
+
+        return $this->storage->store($url, $extension, $blob);
+    }
+
+    /**
+     * Delete all icons except for requested ones.
+     *
+     * @param callable(string):bool $shouldKeep
+     *
+     * @return void
+     */
+    public function cleanup($shouldKeep) {
+        $this->storage->cleanup($shouldKeep);
+    }
+}