Skip to content

Commit

Permalink
helpers\Image: loadImage should not fetch URLs
Browse files Browse the repository at this point in the history 
We need the blob for content type detection so rather than downloading it twice, let's pass it to loadImage directly.
  • Loading branch information
@jtojnar
jtojnar committed May 6, 2020
1 parent 4975e50 commit e187305
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 47 deletions.
45 changes: 32 additions & 13 deletions src/helpers/ContentLoader.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,16 @@ class ContentLoader {
/** @var ThumbnailStore thumbnail store */
private $thumbnailStore;

/** @var WebClient thumbnail store */
private $webClient;

const ICON_FORMAT = Image::FORMAT_PNG;
const THUMBNAIL_FORMAT = Image::FORMAT_JPEG;

/**
* ctor
*/
public function __construct(\daos\Database $database, IconStore $iconStore, Image $imageHelper, \daos\Items $itemsDao, Logger $logger, \daos\Sources $sourcesDao, SpoutLoader $spoutLoader, ThumbnailStore $thumbnailStore) {
public function __construct(\daos\Database $database, IconStore $iconStore, Image $imageHelper, \daos\Items $itemsDao, Logger $logger, \daos\Sources $sourcesDao, SpoutLoader $spoutLoader, ThumbnailStore $thumbnailStore, WebClient $webClient) {
$this->database = $database;
$this->iconStore = $iconStore;
$this->imageHelper = $imageHelper;
Expand All @@ -51,6 +54,7 @@ public function __construct(\daos\Database $database, IconStore $iconStore, Imag
$this->sourcesDao = $sourcesDao;
$this->spoutLoader = $spoutLoader;
$this->thumbnailStore = $thumbnailStore;
$this->webClient = $webClient;
}

/**
Expand Down Expand Up @@ -354,12 +358,20 @@ protected function sanitizeField($value) {
* @return ?string path in the thumbnails directory
*/
protected function fetchThumbnail($url) {
$format = self::THUMBNAIL_FORMAT;
$thumbnailAsJpg = $this->imageHelper->loadImage($url, $format, 500, 500);
if ($thumbnailAsJpg !== null) {
return $this->thumbnailStore->store($url, $thumbnailAsJpg);
} else {
$this->logger->error('thumbnail generation error: ' . $url);
try {
$data = $this->webClient->request($url);
$format = self::THUMBNAIL_FORMAT;
$thumbnailAsJpg = $this->imageHelper->loadImage($data, $format, 500, 500);

if ($thumbnailAsJpg !== null) {
return $this->thumbnailStore->store($url, $thumbnailAsJpg);
} else {
$this->logger->error('thumbnail generation error: ' . $url);
}
} catch (\Exception $e) {
$this->logger->error("failed to retrieve thumbnail $url,", ['exception' => $e]);

return null;
}

return null;
Expand All @@ -373,13 +385,20 @@ protected function fetchThumbnail($url) {
* @return ?string path in the favicons directory
*/
protected function fetchIcon($url) {
$format = Image::FORMAT_PNG;
$iconAsPng = $this->imageHelper->loadImage($url, $format, 30, null);
try {
$data = $this->webClient->request($url);
$format = Image::FORMAT_PNG;
$iconAsPng = $this->imageHelper->loadImage($data, $format, 30, null);

if ($iconAsPng !== null) {
return $this->iconStore->store($url, $iconAsPng);
} else {
$this->logger->error('icon generation error: ' . $url);
}
} catch (\Exception $e) {
$this->logger->error("failed to retrieve image $url,", ['exception' => $e]);

if ($iconAsPng !== null) {
return $this->iconStore->store($url, $iconAsPng);
} else {
$this->logger->error('icon generation error: ' . $url);
return null;
}

return null;
Expand Down
72 changes: 38 additions & 34 deletions src/helpers/Image.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,48 +84,61 @@ public static function getExtension($format) {
*/
public function fetchFavicon($url, $isHtmlUrl = false, $width = null, $height = null) {
// try given url
if ($isHtmlUrl === false) {
$faviconAsPng = $this->loadImage($url, self::FORMAT_PNG, $width, $height);
if ($faviconAsPng !== null) {
return [$url, $faviconAsPng];
}
}

$urlElements = parse_url($url);

// search on base page for <link rel="shortcut icon" url...
$html = null;
try {
$http = $this->webClient->getHttpClient();
$response = $http->get($url);
$html = (string) $response->getBody();
$blob = (string) $response->getBody();
$effectiveUrl = new Uri(WebClient::getEffectiveUrl($url, $response));

if ($response->getStatusCode() !== 200) {
throw new \Exception(substr($html, 0, 512));
throw new \Exception(substr($blob, 0, 512));
}
} catch (\Exception $e) {
$this->logger->debug('icon: failed to get html page: ', ['exception' => $e]);
$this->logger->error("icon: failed to retrieve URL $url", ['exception' => $e]);

return null;
}

if ($isHtmlUrl === false) {
$faviconAsPng = $this->loadImage($blob, self::FORMAT_PNG, $width, $height);
if ($faviconAsPng !== null) {
return [$url, $faviconAsPng];
}
}

if ($html !== null) {
$shortcutIcons = ImageUtils::parseShortcutIcons($html);
// When HTML page, search for icon links
if (preg_match('#^text/html\b#i', $response->getHeaderLine('content-type')) || preg_match('#<html[\s>]#si', $blob)) {
$shortcutIcons = ImageUtils::parseShortcutIcons($blob);
foreach ($shortcutIcons as $shortcutIcon) {
$shortcutIconUrl = (string) UriResolver::resolve($effectiveUrl, new Uri($shortcutIcon));

$faviconAsPng = $this->loadImage($shortcutIconUrl, self::FORMAT_PNG, $width, $height);
if ($faviconAsPng !== null) {
return [$shortcutIconUrl, $faviconAsPng];
try {
$data = $this->webClient->request($shortcutIconUrl);
$faviconAsPng = $this->loadImage($data, self::FORMAT_PNG, $width, $height);

if ($faviconAsPng !== null) {
return [$shortcutIconUrl, $faviconAsPng];
}
} catch (\Exception $e) {
$this->logger->error("failed to retrieve image $url,", ['exception' => $e]);
}
}
}

$urlElements = parse_url($url);

// search domain/favicon.ico
if (isset($urlElements['scheme']) && isset($urlElements['host'])) {
$url = $urlElements['scheme'] . '://' . $urlElements['host'] . '/favicon.ico';
$faviconAsPng = $this->loadImage($url, self::FORMAT_PNG, $width, $height);
if ($faviconAsPng !== null) {
return [$url, $faviconAsPng];
try {
$data = $this->webClient->request($url);
$faviconAsPng = $this->loadImage($data, self::FORMAT_PNG, $width, $height);

if ($faviconAsPng !== null) {
return [$url, $faviconAsPng];
}
} catch (\Exception $e) {
$this->logger->error("failed to retrieve image $url,", ['exception' => $e]);
}
}

Expand All @@ -135,23 +148,14 @@ public function fetchFavicon($url, $isHtmlUrl = false, $width = null, $height =
/**
* Load image from URL, optionally resize it and convert it to desired format.
*
* @param string $url source url
* @param string $data
* @param self::FORMAT_JPEG|self::FORMAT_PNG $format file format of output file
* @param ?int $width target width
* @param ?int $height target height
*
* @return ?string blob containing the processed image
*/
public function loadImage($url, $format = self::FORMAT_PNG, $width = null, $height = null) {
// load image
try {
$data = $this->webClient->request($url);
} catch (\Exception $e) {
$this->logger->error("failed to retrieve image $url,", ['exception' => $e]);

return null;
}

public function loadImage($data, $format = self::FORMAT_PNG, $width = null, $height = null) {
$imgInfo = null;

// get image type
Expand Down Expand Up @@ -208,7 +212,7 @@ public function loadImage($url, $format = self::FORMAT_PNG, $width = null, $heig
try {
$icon = $loader->fromString($data);
} catch (\InvalidArgumentException $e) {
$this->logger->error("Icon “{$url}” is not valid", ['exception' => $e]);
$this->logger->error('Icon is not valid', ['exception' => $e]);

return null;
}
Expand Down

0 comments on commit e187305

Please sign in to comment.