(wip) improving fetching logic

chrome/background.ts

@@ -1,10 +1,12 @@
-import { replenish } from '../src/photo-manager/fetcher';
+import { Replenisher } from '../src/photo-manager/replenisher'
 
 chrome.runtime.onMessage.addListener(function(message) {
   if (message.operation === 'replenishBacklog') {
     const { searchTerms, downloadBatchSize } = message;
 
-    return replenish(searchTerms, downloadBatchSize);
+    const replenisher = new Replenisher(searchTerms);
+    replenisher.downloadBatchSize = downloadBatchSize;
+    replenisher.replenish();
   }
 });
 

src/photo-manager/cleaner.ts

@@ -1,12 +1,12 @@
-import { removePhoto, retrieveAllPhotos, storePhoto } from './fetcher';
+import { storePhoto, retrieveAllPhotos, removePhoto } from "./photos";
 
-export let cleanDownloadFromPhoto = (photo) => {
+export const cleanDownloadFromPhoto = (photo) => {
   photo.blob = null;
   storePhoto(photo);
 };
 
-export let cleanPhotosWithoutGivenSearchTerms = async (searchTerms) => {
-  let photos = await retrieveAllPhotos();
-  let photosToRemove = photos.filter((photo) => photo.searchTerms !== searchTerms);
+export const cleanPhotosWithoutGivenSearchTerms = async (searchTerms) => {
+  const photos = await retrieveAllPhotos();
+  const photosToRemove = photos.filter((photo) => photo.searchTerms !== searchTerms);
   photosToRemove.forEach((photo) => removePhoto(photo));
 };

src/photo-manager/fetch.ts

@@ -0,0 +1,61 @@
+import { clone, take } from 'ramda';
+import { highQualityImageUrlForPhoto, type Photo, type WithSearchTerms } from './photos';
+import resizePhotoBlob from './resizePhoto';
+import shuffle from './shuffle';
+
+export const IMAGE_ENDPOINT_API_URL = 'https://vorfreude-elixir.herokuapp.com/api/images';
+
+export type ImageApiResponse = {
+  photos: {
+    photo: Photo[]
+  }
+};
+
+export function fetchPhotos(photos: Photo[]) {
+  photos = clone(photos);
+
+  return Promise.all(
+    photos.map(async (photo: Photo) => {
+      const url = highQualityImageUrlForPhoto(photo);
+      return fetch(url)
+        .then((response) => response?.blob())
+        .then((blob) => {
+          return {
+            ...photo,
+            blob
+          };
+        });
+    })
+  );
+}
+
+export async function resizePhoto(photo) {
+  const maxLength = 2600;
+  photo.blob = await resizePhotoBlob(photo.blob, maxLength);
+  return photo;
+}
+
+export async function query(searchTerms): Promise<(Photo & WithSearchTerms)[]> {
+  const url = new URL(IMAGE_ENDPOINT_API_URL);
+  url.searchParams.append('search', searchTerms);
+
+  const response = await fetch(url.toString());
+  const result: ImageApiResponse = await response.json();
+  const photos = result.photos.photo;
+
+  return photos
+    .filter(highQualityImageUrlForPhoto)
+    .map((photo) => {
+      return {
+        ...photo,
+        searchTerms: searchTerms
+      };
+    });
+}
+
+export async function fetchPopularPhotoUrl(searchTerms) {
+  const photos = await query(searchTerms);
+  const photoUrls = photos.map(highQualityImageUrlForPhoto);
+
+  return take(50, shuffle(photoUrls)).pop();
+}

src/photo-manager/manager.ts

@@ -1,29 +1,31 @@
 import { take } from 'ramda';
 import {
-  filterForDownloadedPhotos,
   filterPhotosForSearchTerms,
-  filterStalePhotos,
+  isPhotoStale,
   markPhotoAsSeen,
-  shouldDownloadPhotos
+  photoHasDownload,
+  retrieveAllPhotos,
+  shouldDownloadPhotos,
+  storePhoto
 } from './photos';
 
 import { cleanDownloadFromPhoto, cleanPhotosWithoutGivenSearchTerms } from './cleaner';
-
 import shuffle from './shuffle';
-
-import { fetchPopularPhotoUrl, replenish, retrieveAllPhotos, storePhoto } from './fetcher';
-
+import { fetchPopularPhotoUrl } from './fetch';
 import isExtensionEnv from '../helpers/isExtensionEnv';
+import { Replenisher } from './replenisher';
 
 const BATCH_SIZE = 3;
 
 export default class Manager {
   public static urlForBlob = (blob) => URL.createObjectURL(blob);
 
   private searchTerms = '';
+  private replenisher: Replenisher;
 
   setSearchTerms(searchTerms) {
     this.searchTerms = searchTerms;
+    this.replenisher = new Replenisher(searchTerms);
   }
 
   async getPhotos() {
@@ -33,7 +35,7 @@ export default class Manager {
   }
 
   async getPhoto() {
-    const downloadedPhotos = filterForDownloadedPhotos(await this.getPhotos());
+    const downloadedPhotos = (await this.getPhotos()).filter(photoHasDownload);
 
     if (downloadedPhotos.length === 0) {
       const photoUrl = await fetchPopularPhotoUrl(this.searchTerms);
@@ -64,7 +66,8 @@ export default class Manager {
       return;
     }
 
-    take(BATCH_SIZE, filterStalePhotos(photos)).forEach(cleanDownloadFromPhoto);
+    const stalePhotos = photos.filter(isPhotoStale);
+    take(BATCH_SIZE, stalePhotos).forEach(cleanDownloadFromPhoto);
   }
 
   private markPhotoAsSeen(photo) {
@@ -79,7 +82,7 @@ export default class Manager {
         searchTerms: this.searchTerms
       });
     } else {
-      replenish(this.searchTerms);
+      this.replenisher?.replenish();
     }
   }
 }

src/photo-manager/photos.ts

@@ -1,30 +1,65 @@
+import { SimpleIndexedDbAdapter } from '../state/storage/SimpleIndexedDbAdapter';
+
+export type Photo = {
+  id: string;
+  url_o: string;
+};
+
+export type WithSearchTerms = {
+  searchTerms: string;
+}
+
+export type WithBlob = {
+  blob: Blob;
+}
+
+export type WithSeenCount = {
+  seenCount?: number;
+};
+
 const STALE_SEEN_COUNT = 2;
 const ALLOWABLE_STALE_PERCENTAGE = 85;
 const TOTAL_DOWNLOADED = 20;
+const indexStorage = new SimpleIndexedDbAdapter('VORFREUDE_PHOTO_STORAGE');
 
-export let filterForDownloadedPhotos = (photos) => photos.filter((photo) => Boolean(photo.blob));
+export const photoHasDownload = (photo) => Boolean(photo.blob);
+export const isPhotoStale = (photo) => photo.blob && photo.seenCount > STALE_SEEN_COUNT;
 
-export let filterForPreviousDownloadedPhotos = (photos) =>
-  photos.filter((photo) => 'blob' in photo);
+export const shouldDownloadPhotos = (photos) => {
+  const downloadedPhotos = photos.filter(photoHasDownload);
+  const stalePhotos = photos.filter(isPhotoStale);
 
-export let filterStalePhotos = (photos) =>
-  photos.filter((photo) => photo.blob && photo.seenCount > STALE_SEEN_COUNT);
+  const stalePercentage =
+    100 * (stalePhotos.length / downloadedPhotos.length);
 
-export let shouldDownloadPhotos = (photos) => {
-  let stalePercentage =
-    100 * (filterStalePhotos(photos).length / filterForDownloadedPhotos(photos).length);
-  let pastStalePercentage = stalePercentage > ALLOWABLE_STALE_PERCENTAGE;
-  let isUnderTotalDownload = filterForDownloadedPhotos(photos).length <= TOTAL_DOWNLOADED;
+  const pastStalePercentage = stalePercentage > ALLOWABLE_STALE_PERCENTAGE;
+  const isUnderTotalDownload = downloadedPhotos.length <= TOTAL_DOWNLOADED;
 
   return isUnderTotalDownload || pastStalePercentage;
 };
 
-export let filterPhotosForSearchTerms = (photos, searchTerms) =>
+export const filterPhotosForSearchTerms = (photos, searchTerms) =>
   photos.filter((photo) => photo.searchTerms === searchTerms);
 
-export let markPhotoAsSeen = (photo) => {
+export const markPhotoAsSeen = (photo) => {
   photo.seenCount = photo.seenCount ? photo.seenCount + 1 : 1;
   return photo;
 };
 
-export let highQualityImageUrlForPhoto = (photo) => photo.url_o;
+export const highQualityImageUrlForPhoto = (photo) => photo.url_o;
+
+export function retrieveAllPhotos() {
+  return indexStorage.getAll();
+}
+
+export function retrievePhoto(photoId) {
+  return indexStorage.get(photoId);
+}
+
+export function storePhoto(photo) {
+  return indexStorage.set(photo.id, photo);
+}
+
+export function removePhoto(photo) {
+  indexStorage.remove(photo.id);
+}