Skip to content

Commit

Permalink
fix: Resolves Website Thumbnail missing issue
Browse files Browse the repository at this point in the history 
You can now set your own thum.io key to fetch screenshot. More in docs.

closes #131
  • Loading branch information
@towfiqi
towfiqi committed Nov 11, 2023
1 parent c870250 commit 4a60271
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 26 deletions.
19 changes: 16 additions & 3 deletions components/domains/DomainItem.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,31 @@ type DomainItemProps = {
selected: boolean,
isConsoleIntegrated: boolean,
thumb: string,
updateThumb: Function,
}

const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb }: DomainItemProps) => {
const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => {
const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain;
// const router = useRouter();
return (
<div className={`domItem bg-white border rounded w-full text-sm mb-10 hover:border-indigo-200 ${selected ? '' : ''}`}>
<Link href={`/domain/${slug}`} passHref={true}>
<a className='flex flex-col lg:flex-row'>
<div className={`flex-1 p-6 flex ${!isConsoleIntegrated ? 'basis-1/3' : ''}`}>
<div className="domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded border border-gray-200 overflow-hidden">
{thumb && <img src={thumb} alt={domain.domain} />}
<div className="group domain_thumb w-20 h-20 mr-6 bg-slate-100 rounded
border border-gray-200 overflow-hidden flex justify-center relative">
<button
className=' absolute right-1 top-0 text-gray-400 p-1 transition-all
invisible opacity-0 group-hover:visible group-hover:opacity-100 hover:text-gray-600 z-10'
title='Reload Website Screenshot'
onClick={(e) => { e.preventDefault(); e.stopPropagation(); updateThumb(domain.domain); }}
>
<Icon type="reload" size={12} />
</button>
<img
className={`self-center ${!thumb ? 'max-w-[50px]' : ''}`}
src={thumb || `https://www.google.com/s2/favicons?domain=${domain.domain}&sz=128`} alt={domain.domain}
/>
</div>
<div className="domain_details flex-1">
<h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
Expand Down
5 changes: 4 additions & 1 deletion pages/api/settings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ const updateSettings = async (req: NextApiRequest, res: NextApiResponse<Settings
};

export const getAppSettings = async () : Promise<SettingsType> => {
const screenshotAPIKey = process.env.SCREENSHOT_API || '69408-serpbear';
try {
const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' });
const failedQueueRaw = await readFile(`${process.cwd()}/data/failed_queue.json`, { encoding: 'utf-8' });
Expand All @@ -73,6 +74,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL),
available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
failed_queue: failedQueue,
screenshot_key: screenshotAPIKey,
};
} catch (error) {
console.log('Error Decrypting Settings API Keys!');
Expand All @@ -81,7 +83,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
return decryptedSettings;
} catch (error) {
console.log('[ERROR] Getting App Settings. ', error);
const settings = {
const settings: SettingsType = {
scraper_type: 'none',
notification_interval: 'never',
notification_email: '',
Expand All @@ -91,6 +93,7 @@ export const getAppSettings = async () : Promise<SettingsType> => {
smtp_username: '',
smtp_password: '',
scrape_retry: false,
screenshot_key: screenshotAPIKey,
};
const otherSettings = {
available_scapers: allScrapers.map((scraper) => ({ label: scraper.name, value: scraper.id })),
Expand Down
42 changes: 21 additions & 21 deletions pages/domains/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import type { NextPage } from 'next';
import Head from 'next/head';
import { useRouter } from 'next/router';
import { CSSTransition } from 'react-transition-group';
import toast, { Toaster } from 'react-hot-toast';
import TopBar from '../../components/common/TopBar';
import AddDomain from '../../components/domains/AddDomain';
import Settings from '../../components/settings/Settings';
import { useFetchSettings } from '../../services/settings';
import { useFetchDomains } from '../../services/domains';
import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains';
import DomainItem from '../../components/domains/DomainItem';
import Icon from '../../components/common/Icon';

Expand All @@ -23,32 +24,17 @@ const SingleDomain: NextPage = () => {
const { data: domainsData, isLoading } = useFetchDomains(router, true);

useEffect(() => {
// console.log('Domains Data: ', domainsData);
if (domainsData?.domains && domainsData.domains.length > 0) {
const domainThumbsRaw = localStorage.getItem('domainThumbs');
const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
if (domainsData?.domains && domainsData.domains.length > 0 && appSettings?.settings?.screenshot_key) {
domainsData.domains.forEach(async (domain:DomainType) => {
if (domain.domain) {
if (!domThumbs[domain.domain]) {
const domainImageBlob = await fetch(`https://image.thum.io/get/auth/66909-serpbear/maxAge/96/width/200/https://${domain.domain}`).then((res) => res.blob());
if (domainImageBlob) {
const reader = new FileReader();
await new Promise((resolve, reject) => {
reader.onload = resolve;
reader.onerror = reject;
reader.readAsDataURL(domainImageBlob);
});
const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain.domain]: imageBase }));
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: imageBase }));
}
} else {
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domThumbs[domain.domain] }));
const domainThumb = await fetchDomainScreenshot(domain.domain, appSettings.settings.screenshot_key);
if (domainThumb) {
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain.domain]: domainThumb }));
}
}
});
}
}, [domainsData]);
}, [domainsData, appSettings]);

useEffect(() => {
// console.log('appSettings.settings: ', appSettings && appSettings.settings);
Expand All @@ -57,6 +43,18 @@ const SingleDomain: NextPage = () => {
}
}, [appSettings]);

const manuallyUpdateThumb = async (domain: string) => {
if (domain && appSettings?.settings?.screenshot_key) {
const domainThumb = await fetchDomainScreenshot(domain, appSettings.settings.screenshot_key, true);
if (domainThumb) {
toast(`${domain} Screenshot Updated Successfully!`, { icon: '✔️' });
setDomainThumbs((currentThumbs) => ({ ...currentThumbs, [domain]: domainThumb }));
} else {
toast(`Failed to Fetch ${domain} Screenshot!`, { icon: '⚠️' });
}
}
};

return (
<div className="Domain flex flex-col min-h-screen">
{noScrapprtError && (
Expand Down Expand Up @@ -90,6 +88,7 @@ const SingleDomain: NextPage = () => {
selected={false}
isConsoleIntegrated={!!(appSettings && appSettings?.settings?.search_console_integrated) }
thumb={domainThumbs[domain.domain]}
updateThumb={manuallyUpdateThumb}
// isConsoleIntegrated={false}
/>;
})}
Expand All @@ -115,6 +114,7 @@ const SingleDomain: NextPage = () => {
<footer className='text-center flex flex-1 justify-center pb-5 items-end'>
<span className='text-gray-500 text-xs'><a href='https://github.com/towfiqi/serpbear' target="_blank" rel='noreferrer'>SerpBear v{appSettings?.settings?.version || '0.0.0'}</a></span>
</footer>
<Toaster position='bottom-center' containerClassName="react_toaster" />
</div>
);
};
Expand Down
30 changes: 30 additions & 0 deletions services/domains.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,36 @@ export async function fetchDomains(router: NextRouter, withStats:boolean) {
return res.json();
}

export async function fetchDomainScreenshot(domain: string, screenshot_key:string, forceFetch = false): Promise<string | false> {
const domainThumbsRaw = localStorage.getItem('domainThumbs');
const domThumbs = domainThumbsRaw ? JSON.parse(domainThumbsRaw) : {};
if (!domThumbs[domain] || forceFetch) {
try {
const screenshotURL = `https://image.thum.io/get/auth/${screenshot_key}/maxAge/96/width/200/https://${domain}`;
const domainImageRes = await fetch(screenshotURL);
const domainImageBlob = domainImageRes.status === 200 ? await domainImageRes.blob() : false;
if (domainImageBlob) {
const reader = new FileReader();
await new Promise((resolve, reject) => {
reader.onload = resolve;
reader.onerror = reject;
reader.readAsDataURL(domainImageBlob);
});
const imageBase: string = reader.result && typeof reader.result === 'string' ? reader.result : '';
localStorage.setItem('domainThumbs', JSON.stringify({ ...domThumbs, [domain]: imageBase }));
return imageBase;
}
return false;
} catch (error) {
return false;
}
} else if (domThumbs[domain]) {
return domThumbs[domain];
}

return false;
}

export function useFetchDomains(router: NextRouter, withStats:boolean = false) {
return useQuery('domains', () => fetchDomains(router, withStats));
}
Expand Down
3 changes: 2 additions & 1 deletion types.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ type SettingsType = {
scrape_delay?: string,
scrape_retry?: boolean,
failed_queue?: string[]
version?: string
version?: string,
screenshot_key?: string,
}

type KeywordSCDataChild = {
Expand Down

0 comments on commit 4a60271

Please sign in to comment.