From b338eb1db87c9c240109e5083cfd1696af906422 Mon Sep 17 00:00:00 2001 From: Michael Lage Date: Tue, 20 Apr 2021 19:14:19 -0600 Subject: [PATCH] Randomness, Instagram View Stories, and more (#29) * Randomness * fix gatsby/react console errors * Randomness Docs Polish * deleteIndexedDB doc * getCookies, deleteCookies docs * Cookies & Scrapers docs * indexedDB, input, random docs - added missing helpers * instagram nav, auth, stories docs * homepage polish * overview polish * install polish * aborting polish * removing 'export' from code displayed * instagram botactions designed at desktop width - adjust example to set Viewport to desktop width to fix viewStories --- .../components/Layout/styles.js | 2 +- .../gatsby-theme-docs/text/index.mdx | 13 +-- src/config/sidebar.yml | 14 ++- src/docs/api/cookies.mdx | 64 +++++++++-- src/docs/api/indexed-db.mdx | 47 +++++++- src/docs/api/input.mdx | 41 ++++++- src/docs/api/navigation.mdx | 7 +- src/docs/api/random.mdx | 105 ++++++++++++++++++ src/docs/api/scrapers.mdx | 75 ++++++++++++- src/docs/api/time.mdx | 2 +- src/docs/install.mdx | 18 +-- src/docs/overview.mdx | 89 ++++++++------- src/docs/sites/instagram.mdx | 100 ++++++++++++++++- src/docs/sites/linkedin.mdx | 6 +- src/docs/topic/aborting.mdx | 26 +++-- 15 files changed, 502 insertions(+), 107 deletions(-) create mode 100644 src/docs/api/random.mdx diff --git a/src/@rocketseat/gatsby-theme-docs/components/Layout/styles.js b/src/@rocketseat/gatsby-theme-docs/components/Layout/styles.js index b6e61f7..4ffb991 100644 --- a/src/@rocketseat/gatsby-theme-docs/components/Layout/styles.js +++ b/src/@rocketseat/gatsby-theme-docs/components/Layout/styles.js @@ -44,7 +44,7 @@ export const Container = (props) => { - + diff --git a/src/@rocketseat/gatsby-theme-docs/text/index.mdx b/src/@rocketseat/gatsby-theme-docs/text/index.mdx index 4da3256..4d1efd5 100644 --- a/src/@rocketseat/gatsby-theme-docs/text/index.mdx +++ b/src/@rocketseat/gatsby-theme-docs/text/index.mdx @@ -12,20 +12,19 @@ description: 'Build web bots declaratively in TypeScript with the new simple fra [](https://david-dm.org/mrWh1te/Botmation) ![GitHub](https://img.shields.io/github/license/mrWh1te/Botmation) -[Botmation](https://github.com/mrWh1te/Botmation) is a functional TypeScript framework for declaratively building bots with [Puppeteer](https://github.com/puppeteer/puppeteer). It maximizes code *readability*, *maintainability* and *testability* with its compositional design. +[Botmation](https://github.com/mrWh1te/Botmation) is a TypeScript framework for declaratively building bots with [Puppeteer](https://github.com/puppeteer/puppeteer). It maximizes code *readability*, *maintainability* and *testability* with its compositional design. -These bots can do whatever people do with a web browser and more. They can automate social media, take screenshots, make PDF's, scrape, surf the web and much more. πŸŒŠπŸ„πŸ“Έ +These bots can do whatever people do with a web browser and more. They can surf and scrape complex web applications, automate social media, take screenshots, generate website PDF's, and much more! πŸŒŠπŸ„πŸ“Έ ## Features diff --git a/src/config/sidebar.yml b/src/config/sidebar.yml index 155b490..d423303 100644 --- a/src/config/sidebar.yml +++ b/src/config/sidebar.yml @@ -15,18 +15,18 @@ - label: 'Topics' items: - - label: 'Aborting' - link: '/topic/aborting' - - label: 'Piping' - link: '/topic/piping' - - label: 'Injects' - link: '/topic/injects' - label: 'Conditionals' link: '/topic/conditionals' - label: 'Loops' link: '/topic/loops' - label: 'Error Handling' link: '/topic/error-handling' + - label: 'Aborting' + link: '/topic/aborting' + - label: 'Piping' + link: '/topic/piping' + - label: 'Injects' + link: '/topic/injects' - label: 'Concurrency' link: '/topic/concurrency' @@ -67,6 +67,8 @@ link: '/api/navigation' - label: 'Pipe' link: '/api/pipe' + - label: 'Random' + link: '/api/random' - label: 'Scrapers' link: '/api/scrapers' - label: 'Time' diff --git a/src/docs/api/cookies.mdx b/src/docs/api/cookies.mdx index 0856b65..6bd0b57 100644 --- a/src/docs/api/cookies.mdx +++ b/src/docs/api/cookies.mdx @@ -2,11 +2,11 @@ title: 'Cookies' --- -These higher order functions provide a simple way to load and save cookies from and to a Puppeteer page to a local JSON file. +These BotActions provide the means to manage cookies from a Puppeteer page. -You can configure where the cookie JSON files are saved & loaded by using the `BotFileOptions` param as either an inject or as a higher-order param. Higher-order `BotFileOptions` values override injected `BotFileOptions` values. +`BotFileOptions` is used to specify the directory for saving and loading cookies, relative to the bot's executing directory. -These functions are compatible with the higher-order [files()() BotAction](/api/files#files) to inject a customized `BotFileOptions` into each assembled BotAction. +These functions are compatible with the higher-order [files()() BotAction](/api/files#files) to inject a customized `BotFileOptions` into each assembled BotAction. That allows you to standardize the directory for all assembled BotActions that are reading and writing files. ## Save Cookies Saves the cookies from the Puppeteer page in a JSON file with the name provided. @@ -14,15 +14,14 @@ Saves the cookies from the Puppeteer page in a JSON file with the name provided. ```typescript const saveCookies = (fileName: string, botFileOptions?: Partial): BotFilesAction => async(page, options: Partial) => { const hydratedOptions = enrichBotFileOptionsWithDefaults({...options, ...botFileOptions}) - + const cookies = await page.cookies() await fs.writeFile(getFileUrl(hydratedOptions.cookies_directory, hydratedOptions, fileName) + '.json', JSON.stringify(cookies, null, 2)) } ``` `BotFileOptions` is optional, in all cases (higher-order param, as an inject). If nothing is provided, the default is the local directory. -TODO: link vv -See [enrichBotFileOptionsWithDefaults]() for details on setting the directory for files read and created. +See [enrichBotFileOptionsWithDefaults](/api/files#enrichbotfileoptionswithdefaults) for details on setting the directory for files read and created. For an usage example, see the [Instagram example](https://github.com/mrWh1te/Botmation/blob/master/src/examples/instagram.ts), where saving and loading cookies enables the Bot to skip the login flow on subsequent runs, as long as the cookies saved, have not yet expired. @@ -43,7 +42,54 @@ const loadCookies = (fileName: string, botFileOptions?: Partial) ``` `BotFileOptions` is optional, in all cases (higher-order param, as an inject). If nothing is provided, the default is the local directory. -TODO: link vv -See [enrichBotFileOptionsWithDefaults]() for details on setting the directory for files read and created. +See [enrichBotFileOptionsWithDefaults](/api/files#enrichbotfileoptionswithdefaults) for details on setting the directory for files read and created. + +For an usage example, see the [Instagram example](https://github.com/mrWh1te/Botmation/blob/master/src/examples/instagram.ts), where saving and loading cookies enables the Bot to skip the login flow on subsequent runs, as long as the cookies saved, have not yet expired. + +## Get Cookies + +This BotAction returns all cookies for the inputted URL's. When no URL's are specified, it will return the cookies for the current URL only. + +```typescript +const getCookies = (...urls: string[]): BotAction => async(page) => + page.cookies(...urls) +``` + +## Delete Cookies + +This BotAction deletes all cookies provided. + +```typescript +const deleteCookies = (...cookies: Protocol.Network.Cookie[]): BotAction => async(page, ...injects) => { + if (cookies.length === 0) { + if (injectsHavePipe(injects)) { + const pipeValue = getInjectsPipeValue(injects) + if (Array.isArray(pipeValue) && pipeValue.length > 0) { + cookies = pipeValue + } + } + } + + if (cookies.length > 0) { + return page.deleteCookie(...cookies) + } +} +``` + +It's intended to be paired with [getCookies](#getcookies) to designate which cookies are to be deleted. For example: + +```typescript +pipe()( + getCookies(), + deleteCookies() +)(page) +``` +This will grab all cookies, for the current URL, then pipe those cookies into `deleteCookies` for deletion. Otherwise, specify the cookies via the higher-order parameter: + +```typescript +chain( + deleteCookies(cookie1, cookie2, cookie3) +)(page) +``` -For an usage example, see the [Instagram example](https://github.com/mrWh1te/Botmation/blob/master/src/examples/instagram.ts), where saving and loading cookies enables the Bot to skip the login flow on subsequent runs, as long as the cookies saved, have not yet expired. \ No newline at end of file +> If both higher-order params and pipe values are used, this will, like every other BotAction, use the higher-order param instead \ No newline at end of file diff --git a/src/docs/api/indexed-db.mdx b/src/docs/api/indexed-db.mdx index d2f2afb..1888443 100644 --- a/src/docs/api/indexed-db.mdx +++ b/src/docs/api/indexed-db.mdx @@ -104,11 +104,28 @@ const getIndexedDBValue = For an usage example, see the [Instagram isGuest](/sites/instagram#isguest-conditionalbotaction) BotAction. +## Delete Indexed DB + +This BotAction attempts to delete an IndexedDB database, provided the database name. It's possible for this to fail, you can read more about the used underlying logic [here](https://developer.mozilla.org/en-US/docs/Web/API/IDBFactory/deleteDatabase). + +```typescript +const deleteIndexedDB = (databaseName?: string): BotIndexedDBAction => async(page, ...injects) => { + const [, , injectDatabaseName] = unpipeInjects(injects, 2) + + await page.evaluate( + deleteIndexedDBDatabase, + databaseName ?? injectDatabaseName + ) +} +``` + +This BotAction is compatible with the [indexedDBStore()()](#indexeddbstore) for injecting the name of the database for deletion. + ## Helpers -In order to interact with IndexedDB in the Puppeteer Page, separate functions need to be "evaluated" in the Page. Therefore, the following Helper functions are setup as if running inside a Browser Page, instead of NodeJS. +In order to interact with IndexedDB in the Puppeteer Page, separate functions are serialized and evaluated within the browser page. To accomplish this, the following Helper functions were created to be evaluated in the web pages. -> If you `console.log()` in an evaluated Page function, don't look for it in NodeJS's console, but the Puppeteer Page's console. +> If you `console.log()` in an evaluated Page function, the result will appear inside the browser's Console window and *not* the NodeJS terminal. ### setIndexedDBStoreValue() @@ -181,4 +198,30 @@ function getIndexedDBStoreValue(databaseName: string, databaseVersion: number, s } }) } +``` + +### deleteIndexedDBDatabase() + +```typescript +function deleteIndexedDBDatabase(databaseName: string) { + return new Promise((resolve, reject) => { + const DBDeleteRequest = indexedDB.deleteDatabase(databaseName); + + DBDeleteRequest.onerror = function(event) { + logError('delete IndexedDB Database name = ' + databaseName) + event.stopPropagation() + return reject(this.error) + }; + + DBDeleteRequest.onsuccess = function() { + return resolve() + }; + + DBDeleteRequest.onblocked = function(event) { + event.stopPropagation() + logMessage('blocked attempt to delete IndexedDB Database name = ' + databaseName) + return resolve() + }; + }) +} ``` \ No newline at end of file diff --git a/src/docs/api/input.mdx b/src/docs/api/input.mdx index 5890dc9..c9ed91a 100644 --- a/src/docs/api/input.mdx +++ b/src/docs/api/input.mdx @@ -5,13 +5,13 @@ title: 'Input' These BotAction's provide ways to input into a page as User. ## Click -Does a left-mouse click on a HTML element by the provided HTML selector. +Does a left-mouse click on the first HTML element that matches the provided HTML selector. ```typescript const click = (selector: string): BotAction => async(page) => await page.click(selector) ``` -Example: +Example: ```typescript await chain( click('form input[type="submit"]'), @@ -19,6 +19,22 @@ await chain( )(page) ``` +## Click Text +Does a left-mouse click on the first element found whose text content equals the text provided. + +```typescript +const clickText = (text: string): BotAction => + evaluate(clickElementWithText, text) +``` + +Example: +```typescript +chain( + clickText('Save Info') +)(page) +``` +This will click the DOM element with the text "Save Info". + ## Type Type with an imaginary "keyboard" the copy provided. @@ -27,10 +43,25 @@ const type = (copy: string): BotAction => async(page) => await page.keyboard.type(copy) ``` -Example: +Example: ```typescript await chain( - click('form input[name="username"]'), - type('my-username') + click('form input[name="email"]'), + type('example@email.com') )(page) +``` + +## Helpers + +### clickElementWithText + +```typescript +const clickElementWithText = (text: string) => { + const xpath = `//*[text()='${text}']` + const matchingElement = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; + + if (matchingElement instanceof HTMLElement) { + matchingElement.click() + } +} ``` \ No newline at end of file diff --git a/src/docs/api/navigation.mdx b/src/docs/api/navigation.mdx index 0b402cb..32c832d 100644 --- a/src/docs/api/navigation.mdx +++ b/src/docs/api/navigation.mdx @@ -7,16 +7,13 @@ These BotAction's provide simple ways to change the Page's URL. ## Go To Navigates the page to the url provided, unless the active url is the url provided, then it emits a warning in the console. ```typescript -const goTo = (url: string, goToOptions?: Partial): BotAction => +const goTo = (url: string, goToOptions?: Partial): BotAction => async(page) => { - goToOptions = enrichGoToPageOptions(goToOptions) - - // same url check if (page.url() === url) { return } - await page.goto(url, goToOptions) + await page.goto(url, enrichGoToPageOptions(goToOptions)) } ``` `goToOptions` are provided aa they are enriched with safe defaults. See Puppeteer's [Documentation on page.goto()](https://devdocs.io/puppeteer/#pagegotourl-options) for details. diff --git a/src/docs/api/random.mdx b/src/docs/api/random.mdx new file mode 100644 index 0000000..f627823 --- /dev/null +++ b/src/docs/api/random.mdx @@ -0,0 +1,105 @@ +--- +title: 'Random' +--- + +These BotActions focus on randomness. + +## Random Decimal +This higher-order BotAction [injects](/api/inject) the inputted random decimal function, as the first inject, to override the default random generating decimal function for all assembled BotActions. + +> You can override the default random function for each Random BotAction through their own optional higher-order parameter `overloadGenerateRandomDecimal`. The higher-order param will override the injected function, when both are provided. + +```typescript +const randomDecimal = (injectGenerateRandomDecimalFunction: NumberReturningFunc) => + (...actions: BotAction[]): BotAction => + inject(injectGenerateRandomDecimalFunction)( + errors('randomDecimal()()')(...actions) + ) +``` + +> The default random generating function is the pseudo [Math.random()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random) function + +In this example, `probably()()` will use `cryptoBasedRandomDecimal` function injected by `randomDecimal()()`: + +```typescript +import { cryptoBasedRandomDecimal } from './crypto-based-random-function' + +randomDecimal(cryptoBasedRandomDecimal)( + probably()( + likePost + ) +)(page) +``` + +## Roll Dice +This BotAction rolls a dice with X number of sides. If the rolled dice lands on 1, it runs the assembled BotActions. By default, the dice has only 1 side, therefore it has a 100% chance of running assembled BotActions, without input. + +> Limits choice of randomness to less than or equal to 50% + +```typescript +const rollDice = + (numberOfDiceSides = 1, overloadGenerateRandomDecimal?: NumberReturningFunc) => + (...actions: BotAction[]): BotAction => + probably(1 / numberOfDiceSides, overloadGenerateRandomDecimal)(...actions) +``` +In this example, we are rolling a 3 sided dice: + +```typescript +chain( + rollDice(3)( // 1 in 3 odds + likePost + ) +)(page) +``` + +It's possible to override the `generateRandomDecimal()` function with your own via the optional higher-order parameter `overloadGenerateRandomDecimal` or as the first inject. + +## Probably + +This BotAction runs assembled BotActions if the randomly generated decimal is less than or equal to the `probability` provided. Unlike, `rollDice()()`, `probably()()` supports probabilities between 0 and 100%. + +> The default probability is 60% + +```typescript +const probably = + (probability = .6, overloadGenerateRandomDecimal?: NumberReturningFunc) => + (...actions: BotAction[]): BotAction => + async(page, ...injects) => { + if (!overloadGenerateRandomDecimal) { + const [,injectedRandomDecimalFunction] = unpipeInjects(injects, 1) + + if (typeof injectedRandomDecimalFunction === 'function') { + overloadGenerateRandomDecimal = injectedRandomDecimalFunction // once injects becomes Map based :) + } else { + overloadGenerateRandomDecimal = generateRandomDecimal + } + } + + if (overloadGenerateRandomDecimal() <= probability) { + return assemblyLine()(...actions)(page, ...injects) + } + } +``` + +Probabilities are expressed as a decimal. In this example, there is an 80% chance of running `likePost`: + +```typescript +chain( + probably(.8)( // 80% chance + likePost + ) +)(page) +``` + +It's possible to override the `generateRandomDecimal()` function with your own via the optional higher-order parameter `overloadGenerateRandomDecimal` or as the first inject. + +## Helpers + +### generateRandomDecimal() + +The default random decimal function returns the value generated from pseudo `Math.random()`. + +```typescript +const generateRandomDecimal = (): number => + Math.random() +``` \ No newline at end of file diff --git a/src/docs/api/scrapers.mdx b/src/docs/api/scrapers.mdx index 54cae7f..7f3d137 100644 --- a/src/docs/api/scrapers.mdx +++ b/src/docs/api/scrapers.mdx @@ -2,13 +2,13 @@ title: 'Scrapers' --- -These BotAction's scrape the Page's `document` by doing something within the Page's context. +These BotActions interact with the page through evaluated serialized code. They can scrape or manually run code within the page's context. -Scrapers use a HTML parser to convert the `outerHTML` property of an HTML Element(s) into an interactive object. +The scraper BotActions use a HTML parser to convert the `outerHTML` property of an HTML Element(s) into an interactive object. The default HTML parser is [cheerio's](https://github.com/cheeriojs/cheerio) [load() function](https://github.com/cheeriojs/cheerio#loading). -You can override the HTML parser by providing a function to operate on the `outerHTML` via the optional higher-order param `higherOrderHTMLParser`, or the optional first [inject](/topic/injects). If both are provided, the higher-order param will override the injected one, so if you want to use multiple, feel free too. +> Override the default HTML parser by providing your own function to parse the `outerHTML` via higher-order param `higherOrderHTMLParser` or the first [inject](/topic/injects). When both are present, the higher-order function will be used instead of the injected one. ## HTML Parser This higher-order BotAction will [inject](/topic/injects) the provided HTML parser function of the first `htmlParser()` call, into the assembled BotAction's of the second call `htmlParser()()`. @@ -27,7 +27,16 @@ const htmlParser = (htmlParser: Function) => ) ``` -For an usage example, see the [LinkedIn example](https://github.com/mrWh1te/Botmation/blob/master/src/examples/linkedin.ts). +Example: +```typescript +await pipe()( + htmlParser(customHTMLParser)( + $('html body .user') + ) +)(page) +``` + +The assembled `$()` scraper will use the `customHTMLParser` function instead of the default. ## Scrape Single This ScraperBotAction will parse the HTML of the first Element in the document that matches the provided selector. @@ -115,15 +124,56 @@ await pipe()( ## Evaluate -This BotAction isn't a Scraper but an important building block for scraping. It's a simple BotAction that wraps Puppeteer Page's [evaluate()](https://devdocs.io/puppeteer/index#pageevaluatepagefunction-args) method to execute JavaScript in the Page. It takes a function, and parameters and returns what the function returns. +This BotAction isn't a ScraperBotAction, but can be used to scrape and more. It's a simple BotAction that wraps Puppeteer Page's [evaluate()](https://devdocs.io/puppeteer/index#pageevaluatepagefunction-args) method to execute serialized JavaScript in the Page. ```typescript const evaluate = (functionToEvaluate: EvaluateFn, ...functionParams: any[]): BotAction => async(page) => await page.evaluate(functionToEvaluate, ...functionParams) ``` +The `functionToEvaluate` is serialized by Puppeteer then injected into the context of the Page to be evaluated there. + +> Do not reference 3rd party libraries inside evaluated functions since they won't be available in the Page's context. + For an example, see the Navigation BotAction [scrollTo()](/api/navigation#scroll-to) +## Text Exists + +This Conditional BotAction checks the Puppeteer page for the text provided. If found, it returns `true` otherwise `false`. + +> Checks the `document.documentElement` for either `textContent` and `innerText` matching text provided + +```typescript +const textExists = (text: string): ConditionalBotAction => + evaluate(textExistsInDocument, text) +``` + +For example: +```typescript +givenThat(textExists('Do you want to save the password?'))( + clickNo +) +``` + +## Element Exists + +This Conditional BotAction checks the Puppeteer page DOM for a HTML Node element that matches the selector provided. If found, it returns `true` otherwise `false`. + +```typescript +const elementExists = (elementSelector: string): ConditionalBotAction => + evaluate(elementExistsInDocument, elementSelector) +``` + +For example: +```typescript +givenThat(elementExists('body .user .notifications'))( + $('body .user .notifications'), + log('Number of User Notifications') +) +``` + +This will only attempt to scrape and log the User's notifications, if that element is found within the DOM. + ## Helpers In order to "scrape" the page's document's HTML, these Helper functions are evaluated in the page's context. To avoid race conditions with the document's nodes, the `outerHTML` property (string representation of the HTML element and children) is returned to the NodeJS context than parsed with a HTML parser. @@ -140,4 +190,19 @@ const getElementOuterHTML = (htmlSelector: string): string|undefined => ```typescript const getElementsOuterHTML = (htmlSelector: string): string[] => Array.from(document.querySelectorAll(htmlSelector)).map(el => el.outerHTML) +``` + +### textExistsInDocument() + +```typescript +const textExistsInDocument = (text: string): boolean => + ( document.documentElement.textContent || document.documentElement.innerText ) + .indexOf(text) > -1 +``` + +### elementExistsInDocument() + +```typescript +const elementExistsInDocument = (htmlSelector: string): boolean => + document.querySelector(htmlSelector) !== null ``` \ No newline at end of file diff --git a/src/docs/api/time.mdx b/src/docs/api/time.mdx index b7714a9..656a5a8 100644 --- a/src/docs/api/time.mdx +++ b/src/docs/api/time.mdx @@ -26,7 +26,7 @@ Schedule assembled BotActions for a single event in the future or a re-occuring > Similar to the BotActions from [Loops](/api/loops) and [Branching](/api/branching), the Schedule BotAction will run assembled BotActions in a [pipe](/api/assembly-lines#pipe). ```typescript -export const schedule = +const schedule = (schedule: string|Date) => (...actions: BotAction[]): BotAction => async(page, ...injects) => { diff --git a/src/docs/install.mdx b/src/docs/install.mdx index 2cffcba..90ddcd4 100644 --- a/src/docs/install.mdx +++ b/src/docs/install.mdx @@ -6,18 +6,18 @@ Botmation is written in [TypeScript](https://www.typescriptlang.org/) and runs o ## Bare bones setup -If you're starting a new project, in your project's code directory, use `npm` from NodeJS and `tsc` from TypeScript, to initialize the environment: +If you're starting a blank new project, from your project's code directory, use `npm` from NodeJS and `tsc` from TypeScript, to initialize the environment: ```bash npm init -y tsc --init touch index.ts ``` -Now you can begin writing bot code in `index.ts` file and more. +Now you can begin writing your bot code in `index.ts` file. ## Install Botmation -For any bot work, you'll need at least the core Botmation package, installed via `npm`: +For any bot work, you'll need at least the core Botmation package. Install via `npm`: ```bash npm install --save @botmation/core @@ -25,7 +25,7 @@ npm install --save @botmation/core ## Install Puppeteer -If you haven't already, install puppeteer: +If you haven't already, install [Puppeteer](https://pptr.dev/): ```bash npm install --save puppeteer @@ -33,19 +33,21 @@ npm install --save puppeteer ## Import BotActions -Then in your code, import the Botmations, helpers, etc from the `@botmation/core` package: +Import BotActions, helpers, etc from the `@botmation/core` package: ```typescript import { chain, goTo, screenshot } from '@botmation/core'; ``` All of Core's functions are documented with examples, under Core in the sidebar on the left. +> Helpers are plain functions to reduce boilerplate. They are not BotActions. They are documented in their respective API pages. + ## Compile and run -If you're following the [New project setup](#new-project-setup), compile and run the code: +If you're following the [Bare bones setup](#bare-bones-setup), compile and run the code: ```bash -tsc # compiles -node index.js # runs +tsc # compile +node index.js # run ``` > If you want to watch Puppeteer control the browser, configure it with `headless: false`. Otherwise, the browser window won't display when the bot runs. diff --git a/src/docs/overview.mdx b/src/docs/overview.mdx index 7f437d9..c0272ec 100644 --- a/src/docs/overview.mdx +++ b/src/docs/overview.mdx @@ -8,41 +8,47 @@ Botmation is a simple, declarative framework to build bots with *composable* fun

β€œEverything should be made as simple as possible, but no simpler.”
-

~ Albert Einstein
+ ~ Albert Einstein

Orange Bot -**BotActions** are self-reliant, async functions that perform bot tasks from specific things such as "click this button", to broad flows such as "scrape this feed and like my friends' posts". They can be ran by themselves or composed together as an assembly of diverse functions that handle complex tasks. They are like bricks, you can lay them down to build walls, then use the walls to build buildings and so forth. Except, at each level of composition (brick, wall, building, etc), they are still considered the same *type* of thing. Therefore, they can be mixed and used anywhere, regardless of their compositional complexity. +BotActions are self-reliant, single-purposed, async functions that perform tasks from simple, such as "click this button", to broad complex flows, such as "scrape this feed and like all of my friends' posts". They are ran individually or assembled together in a new BotAction. -Imagine a door πŸšͺ into a coffee shop β˜• where customers 🧍,🧍,🧍 enter. As they step inside they form a line 🧍🧍🧍 to place their orders. The first customer served, is the first customer in line, then the following, and so forth. +> BotActions are like bricks. Bricks build walls,, walls build buildings, buildings build cities and so forth. Except, at each level of composition (brick, wall, building, etc), they are *all* the same *type* of function, a BotAction. Regardless of their compositional complexity, they are all single async functions called BotActions. -Botmation's bots are assembled in lines of **BotActions**. The bots run the actions in the order declared, from first to last. +Imagine a door πŸšͺ into a coffee shop β˜• where customers 🧍,🧍,🧍 enter. As they step inside, a customer line forms 🧍🧍🧍. The first customer served, is the first customer in line, then the following, and so forth. -However, a single **BotAction** can actually be a composition of a bunch of other **BotActions**. Therefore, a single person 🧍 in the coffee shop β˜•, can actually be a whole other line of people 🧍🧍🧍. Then any one of those people 🧍 in this "sub" line 🧍🧍🧍, can be a whole other line of people 🧍🧍🧍, and infinitely deep β™ΎοΈπŸ‡. +Botmation's bots are *assembled in lines* of BotActions. Assembled bots run their actions in the order declared, from first to last. + +A single BotAction can be an assembled line of other BotActions. Therefore, a single person 🧍 in the coffee shop β˜•, can actually be a whole other line of people 🧍🧍🧍. Then any one of those people 🧍 in this "sub" line 🧍🧍🧍, can be a whole other line of people 🧍🧍🧍 infinitely deep β™ΎοΈπŸ‡. The possibilities are endless! -## Running BotActions +> If you're familiar with de-coupling functionality, this compositional pattern creates structure to keep all units of functionality separated, pluggable and 100% reusable. -BotActions run on a [Puppeteer page instance](https://pptr.dev/#?product=Puppeteer&version=v7.0.4&show=api-class-page) and are completed by resolving their function's returned Promise. +## Running BotActions -> If you're unfamiliar with async functions in JavaScript, they are simply functions that return Promises. Promises are a way to handle async functionality like reacting to events, when you don't know when they'll take place. This is important since a lot of Puppeteer's API is asynchronous. +BotActions wrap a Puppeteer [Page](https://pptr.dev/#?product=Puppeteer&show=api-class-page) instance as a function param. They are completed once their returned promise resolves. -Some BotActions are have higher-order functions to customize their functionality. For example, the BotAction [goTo()](/api/navigation/#go-to) uses a higher-order function to set the URL for the bot to navigate too, while [waitForNavigation](/api/navigation/#wait-for-navigation) does not: +Some BotActions have higher-order functions that customize their functionality. For example, the BotAction [goTo()](/api/navigation/#go-to) uses a higher-order function to set the URL for the bot to navigate too, while [waitForNavigation](/api/navigation/#wait-for-navigation) does not: ```typescript const page = await browser.newPage() // Puppeteer Browser -await goTo('https://example.com')(page) // call higher-order than BotAction -await waitForNavigation(page) // no higher-order, just call BotAction +await goTo('https://example.com')(page) // call higher-order to get the BotAction +await waitForNavigation(page) // no higher-order, just resolve the BotAction ``` -> Higher-order functions are functions that return a function. Botmation uses higher-order functions to customize BotActions during runtime. +> [Higher-order functions](https://en.wikipedia.org/wiki/Higher-order_function) are functions that return a function. Botmation uses higher-order functions to customize BotActions during runtime. -## Building Bots +So how is this better than using Puppeteer directly? BotActions can be assembled into new reusable functions, of varying compositional complexity. No matter what, regardless of their complexity, they are *just* single async functions. -Bots are built by assembling BotActions in a line. Let's complete the code above, by assembling it into a bot with a special kind of BotAction, an [Assembly Line](/api/assembly-lines). The simplest is called [chain()()](/api/assembly-lines#chain): +## Assembling Bots + +Bots are built by assembling BotActions, declaratively, in a line. This makes each bot's code highly readable. + +Let's edit the code from the example above, by assembling it into a reusable bot via a special kind of BotAction, an [Assembly Line](/api/assembly-lines). The simplest is called [chain()()](/api/assembly-lines#chain): ```typescript const bot = chain( @@ -51,9 +57,9 @@ const bot = chain( ) ``` -This bot is assembled by [chaining](/api/assembly-lines#chain) BotActions together. +This BotAction [chains](/api/assembly-lines#chain) BotActions together, in a single line, as a new reusable BotAction. -> This is similar to [Currying](https://en.wikipedia.org/wiki/Currying) in Functional Programming. +> This technique is inspired by [Currying](https://en.wikipedia.org/wiki/Currying) from Functional Programming. Now let's run the bot: ```typescript @@ -62,19 +68,23 @@ const bot = chain( waitForNavigation ) -const page = await browser.newPage() // Puppeteer.Browser -await bot(page) // run with this Puppeteer page +const page = await browser.newPage() +await bot(page) ``` -We can run this bot code on multiple Puppeteer pages. We can even run bots [concurrently](/topic/concurrency). +This bot function can be reused on multiple Puppeteer pages. We can even run multiple bots [concurrently](/topic/concurrency), using the same code. + +> An assembled bot is still a BotAction. But, a BotAction is a bot part. Philosophically speaking, parts and bots are differentiated via observation. BotActions are bot parts, until they are ran as a bot, either individually or in an assembled composition. -> An assembled bot is still a BotAction. But, a BotAction is a bot part. Therefore, philosophically, parts and bots are differentiated by observation. BotActions are bot parts, until they are ran as a bot, either individually or in a composition. +So how do we create our own BotActions that go beyond the functionality provided by Botmation? -## Making Simple BotActions +## Simple BotActions -There's three main styles of BotAction code, from simplest to most complex. +There's three main ways to make your own BotActions, from simple to complex. -The simplest kind have no higher order functions. They are simply async functions. Let's take a look at a familiar example, from [Navigation](/api/navigation), called [waitForNavigation](/api/navigation/#wait-for-navigation): +> Every BotAction receives a Puppeteer `page` as its first param + +The simplest are plain async functions. Here's the code for the [waitForNavigation](/api/navigation/#wait-for-navigation) BotAction: ```typescript const waitForNavigation: BotAction = async(page) => { @@ -82,38 +92,39 @@ const waitForNavigation: BotAction = async(page) => { } ``` -There are no higher-order functions wrapping this BotAction. It is a single async function that takes a Puppeteer `page` param to operate on. Simplicity is great. +This BotAction calls the `page` [waitForNavigation](https://pptr.dev/#?product=Puppeteer&show=api-pagewaitfornavigationoptions) function. Simplicity is great. + +> You can call any Puppeteer page function inside a BotAction. However, you're not limited too just Puppeteer. You can [inject](/topic/injects) other libraries into your bots, which helps with writing tests for your BotActions, as they can easily be mocked. -## Making Dynamic BotAction's +## Dynamic BotActions -BotActions can be customizable by wrapping them in a higher-order function, to provide customizing values for the async functionality. Let's take a closer look at [goTo()](/api/navigation/#go-to), from "Navigation" that navigates the page to the URL param: +BotActions wrapped in a higher-order functions are dynamic. Let's look at the code for the [goTo()](/api/navigation/#go-to) BotAction that navigates the page to the URL provided: ```typescript -const goTo = (url: string, goToOptions?: Partial): BotAction => +const goTo = (url: string, goToOptions?: Partial): BotAction => async(page) => { - goToOptions = enrichGoToPageOptions(goToOptions) - - // same url check if (page.url() === url) { return } - await page.goto(url, goToOptions) + await page.goto(url, enrichGoToPageOptions(goToOptions)) } ``` -[goTo()](/api/navigation/#go-to) uses a higher-order sync function with two params, `url` and `goToOptions?` that customize the returned BotAction function. +The higher-order sync function returns the customized BotAction function during runtime. Here [goTo()](/api/navigation/#go-to) uses a higher-order sync function to dynamically set the `url` and `goToOptions?` for the BotAction. -The higher order parameters can be whatever you need them to be. They're typed as a spread array of `any`, so add more if you need more. Also, you are not limited to one higher-order sync function. Stack them up, as high as need be! The possibilities are endless. If you're curious, check out the [Loops BotActions](/api/loops) for practical BotActions that wrap themselves with two higher order functions. +Higher order function parameters can be whatever you need them to be. They're typed as a spread array of `any`, so add more if you need more. Also, you are not limited to one higher-order sync function. Stack them up, as high as need be! The possibilities are endless. -## Composing BotAction's +> Feeling curious about stacked higher-order functions? Check out the [Loops BotActions](/api/loops) for practical BotActions that wrap themselves with two higher order functions. -BotActions with higher-order functions can be used to create static, more readable BotActions: +Dynamic BotActions can be reused to create static, more readable BotActions: ```typescript const goToGoogle = goTo('https://google.com') ``` -However, the best has been saved for last! [Assembly Line BotActions](/api/assembly-lines) can be used to compose complex BotActions that handle broad flows that can be reused across all your bots. For example, how about a single BotAction that logs a bot into a website through a Login form? +## Composed BotActions + +However, the best has been saved for last! The technique for assembling bots is the same way for assembling complex BotActions. Remember, they are all BotActions until ran. For example, here is a composed BotAction that logs a bot into a website through a Login form: ```typescript const login = (username: string, password: string): BotAction => @@ -129,6 +140,8 @@ const login = (username: string, password: string): BotAction => ) ``` -`login()` is a higher-order sync function that assembles BotActions with the [Chain BotAction](/api/assembly-lines#chain). The first call of [chain()](/api/assembly-lines#chain) is a regular synchronous function that returns a customized BotAction. +`login()` is a higher-order sync function that returns an assembled line of BotActions using the [Chain](/api/assembly-lines#chain) BotAction. + +This BotAction completes a login flow for an example website form. Once started, it runs the assembled BotActions, one at a time, in the line they are declared. First, it navigates to the login page, then enters the `username` and `password` into the login page's form's inputs, submits the form, waits for the Navigation of the page to complete, before finally logging `Login Complete` in the NodeJS console, but you already know that from reading the code! -This practical BotAction completes a login flow for a common website form. It starts by going to the login page, entering the `username` and `password` into the login page's form's inputs, then submits the form, and finally waits for the Navigation of the page to complete, before logging `Login Complete` in the NodeJS console. +> Assembly Lines are not the only kind of BotAction that assembles other BotActions. For more complex situations, check out [Branching](/api/branching) & [Loop](/api/loops) BotActions for unique ways to assemble BotActions i.e. [if statement](/api/branching#given-that). diff --git a/src/docs/sites/instagram.mdx b/src/docs/sites/instagram.mdx index 65ec6ad..5ab96eb 100644 --- a/src/docs/sites/instagram.mdx +++ b/src/docs/sites/instagram.mdx @@ -2,13 +2,11 @@ title: Instagram --- -These functions focus on Instagram's web app. - ## Overview -The current set of BotActions facilitate the login flow. Here is a working [example](https://github.com/mrWh1te/Botmation/blob/master/apps/bot-instagram). +These functions focus on Instagram's web app, at desktop dimensions. Here is a working [example](https://github.com/mrWh1te/Botmation/blob/master/apps/bot-instagram). -> These are all included in the [@botmation/instagram](https://www.npmjs.com/package/@botmation/instagram) package. +> These BotActions are designed in Instagram's web app running at desktop widths. The accessibility of some features differ in mobile then in desktop. ## Install @@ -17,7 +15,39 @@ In addition to installing [@botmation/core](/install), install [@botmation/insta npm i -s @botmation/instagram ``` -Now you can import any of these BotActions or selectors listed on this page, along with any other BotActions listed. +All BotActions, helpers, and selectors listed on this page, are exported by that package. + +## Navigation + +These BotActions focus on changing the URL of the Puppeteer page to specific locations within Instagram's web app. + +### goToHome + +```typescript +const goToHome: BotAction = + goTo('https://www.instagram.com/') +``` + +### goToMessaging + +```typescript +const goToMessaging: BotAction = + goTo('https://www.instagram.com/direct/inbox/') +``` + +### goToExplore + +```typescript +const goToExplore: BotAction = + goTo('https://www.instagram.com/explore/') +``` + +### goToSettings + +```typescript +const goToSettings: BotAction = + goTo('https://www.instagram.com/accounts/edit/') +``` ## Auth @@ -64,6 +94,66 @@ const isLoggedIn: ConditionalBotAction = ``` This BotAction is a composition that uses [indexedDBStore()()](/api/indexed-db#indexeddbstore) to assemble BotAction's with the IndexedDB Database name and Store name [injected](/topic/injects). This simply grabs the value for the `users.viewerId` key then [maps](/api/pipe#map) it to the corresponding boolean value. +### logout + +This function deletes cookies and local storage that include the user's session information, to effect a logout, upon page reload. + +```typescript +const logout: BotAction = pipe()( + getCookies(), + deleteCookies(), + clearAllLocalStorage, + reload() +) +``` + +### isSaveYourLoginInfoActive + +This BotAction tests the page for the "Save Your Login Info" flow that sometimes appears after initial login. + +```typescript +const isSaveYourLoginInfoActive: ConditionalBotAction = + textExists('Save Your Login Info?') +``` + +### clickSaveYourLoginInfoYesButton + +This will close out the "Save Your Login Info" flow affirmatively. + +```typescript +const clickSaveYourLoginInfoYesButton: BotAction = + clickText('Save Info') +``` + +### clickSaveYourLoginInfoNoButton + +This will close out the "Save Your Login Info" flow negatively. + +```typescript +const clickSaveYourLoginInfoNoButton: BotAction = + clickText('Not Now') +``` + +## Stories + +These BotActions interact with the popular Instagram feature: Stories. + +### viewStories + +This BotAction, when ran on the Home page, will view all available Stories from first to last, until complete. + +```typescript +const viewStories: BotAction = + givenThat(elementExists(FIRST_STORY + ' button'))( + click(FIRST_STORY + ' button'), + wait(1000), + forAsLong(elementExists(STORIES_VIEWER_NEXT_STORY_ICON))( + click(STORIES_VIEWER_NEXT_STORY_ICON), + wait(500) + ) + ) +``` + ## Modals These BotAction's help handle Instagram's Modals. diff --git a/src/docs/sites/linkedin.mdx b/src/docs/sites/linkedin.mdx index d204d5e..764d5e6 100644 --- a/src/docs/sites/linkedin.mdx +++ b/src/docs/sites/linkedin.mdx @@ -199,11 +199,11 @@ By default, when someone logs into the LinkedIn web app, the "Messaging" center Helpful HTML element selectors in the LinkedIn web app: ```typescript // Selectors for Messaging Overlay -export const messagingOverlayHeaderSelector = 'header.msg-overlay-bubble-header' +const messagingOverlayHeaderSelector = 'header.msg-overlay-bubble-header' // Selectors for the main News Feeds -export const feedPostsSelector = '.application-outlet .feed-outlet [role="main"] div[data-id]' -export const feedPostAuthorSelector = '.feed-shared-actor__title' +const feedPostsSelector = '.application-outlet .feed-outlet [role="main"] div[data-id]' +const feedPostAuthorSelector = '.feed-shared-actor__title' ``` ## Helpers diff --git a/src/docs/topic/aborting.mdx b/src/docs/topic/aborting.mdx index 2f47045..948c8a2 100644 --- a/src/docs/topic/aborting.mdx +++ b/src/docs/topic/aborting.mdx @@ -4,7 +4,7 @@ title: Aborting ## AbortLine Signal -Sometimes it's necessary to abort an assembly line(s) of BotAction's. Perhaps something is removed from a web page or a condition wasn't met that subsequent BotAction's rely on. When this happens, a BotAction returns an `AbortLineSignal`. +Sometimes it's necessary to abort an assembled line of BotActions. Perhaps the bot detects that a required button, to complete its task, is missing from the web page. Therefore, instead of letting the subsequent BotActions run and fail, from trying to click a button that doesn't exist, a BotAction can instead return an `AbortLineSignal`. ```typescript type AbortLineSignal = { @@ -14,23 +14,25 @@ type AbortLineSignal = { } ``` -AbortLine signals are safely created with the [createAbortLineSignal()](/api/abort#createabortlinesignal) helper. It's a simple JSON object with potentially two to three properties. +AbortLine signals are created with the [createAbortLineSignal()](/api/abort#createabortlinesignal) helper. As seen above, it's a simple JSON object with two to three properties. ### Assembled Lines Count -An AbortLine signal's important property is called `assembledLines`. It informs the assembler how many lines of assembly to break. One is one line, two is two, three is three and so forth. +The most important property of an AbortLine Signal is called `assembledLines`. It informs the higher-order assembler, how many assembled lines of BotActions, the signal should break. + +When as assembler processes an AbortLine Signal, with a positive `assembledLines` count, it breaks its sub-line, then decreases the count by 1 before returning the signal for the next assembler to process. The default count of `assembledLines` is one for both the [abort()](/api/abort#abort) BotAction and the [createAbortLineSignal()](/api/abort#createabortlinesignal) helper. It can be any positive number, including zero. ### Infinity AbortLine Signal -It's possible to abort all lines of assembly without knowing how many there are by returning an Infinity AbortLine signal. +It's possible to abort all lines of assembly without knowing how many there are by returning an Infinity AbortLine Signal. -Infinity AbortLine signals have `assembledLines` set to `0`. When an assembler processes an Infinity AbortLine signal, it aborts what it's doing and returns the same signal without change. +Infinity AbortLines have `assembledLines` set to `0`. When an assembler processes an Infinity AbortLine signal, it aborts what it's doing and returns the same signal without decreasing the `assembledLines` count. ### Optional Pipe Value `AbortLineSignal` has an optional param called `pipeValue` that carries a `PipeValue` back to the final aborted assembler for return. -> The optional `pipeValue` param does not get returned by a [Chain](/api/assembly-lines#chain), when it's the final assembler to process an AbortLine signal. +> The optional `pipeValue` param does not get returned by [Chain](/api/assembly-lines#chain), when it's the final assembler to process an AbortLine signal, because chains don't deal with pipe values. Use [Pipe](/api/assembly-lines#pipe) instead. ## Process AbortLine Signal @@ -42,15 +44,15 @@ Here is a table for the default assemblers' aborting behavior: | - | - | | 0 | Infinity AbortLine Signal aborts every BotAction and returns itself without change on each abort | | 1 | Default AbortLine Signal aborts only one line of assembled BotActions and returns the abortLineSignal.pipeValue | -| 2+ | Level specific AbortLine Signal aborts one line of assembled BotActions at a time while reducing its assembledLines count to 1 for final abort | +| 2+ | Level specific AbortLine Signal aborts one line of assembled BotActions at a time, for each count of `assembledLines` | -Each assembler will detect the value returned by the resolved BotAction for an AbortLine signal. If it is one, it processes it to the effect listed above. +Each assembler will detect the value returned by the resolved BotAction for an AbortLine Signal. If it is an AbortLine Signal, it processes it to the effect listed above. -This is the default aborting behavior. There are exceptions for some BotAction's such as the [Loops](/api/loops), [Branching](/api/branching), [Switch Pipe](/api/assembly-lines#switch-pipe), and [Pipe Case(s)](/api/pipe#pipe-case), which implement their own unique aborting behavior. +There are exceptions for some BotActions such as the [Loops](/api/loops), [Branching](/api/branching), [Switch Pipe](/api/assembly-lines#switch-pipe) and [Pipe Case(s)](/api/pipe#pipe-case). They implement slightly different behavior, to match their unique functionality to offer developers more control. ## Unique Aborting behavior -[Switch Pipe](/api/assembly-lines#switch-pipe) implements its own AbortLine signal behavior in order to support its own unique functionality. When it hasn't received a [CasesSignal](/topic/conditionals/#cases-signal) with its condition passed, it ignores default AbortLine signals. That's because [abort()](/api/abort#abort) is treated like `break;` from the traditional `switch(){}` code block, inside a [Switch Pipe](/api/assembly-lines#switch-pipe). +[Switch Pipe](/api/assembly-lines#switch-pipe) was created to supply a switch/case/break flow in Botmation. To accomplish that, it implements its own AbortLine Signal behavior, which treats the [Abort](/api/abort#abort) BotAction like a `break;` statement in a [switch case](https://www.w3schools.com/js/js_switch.asp). When a Switch Pipe processes its assembled BotActions, and hasn't received a [CasesSignal](/topic/conditionals/#cases-signal) with its condition passed, it swallows up AbortLine Signals with one `assembledLines`, effectively ignoring them. By assembling [Pipe Case(s)](/api/pipe#pipe-case) BotActions whose conditions pass, the Switch Pipe will STOP ignoring Abort BotActions. Therefore, mimicing the switch/case/break flow functionally. Here's an [example](/sites/linkedin#like-user-posts-from). Once a `pipeCase` or `pipeCases` matches, the next `abort()` statement will stop remaining assembled BotActions from running. -Some BotActions have unique AbortLine signal behavior for aborting parts of the unique assemblers. For example, a [For All](/api/loops#for-all) BotAction's loop iteration (runs an assembled line of BotActions) can abort out of its loop iteration with a default AbortLine signal or abort the entire loop with two or more `assembledLines`. +The [For All](/api/loops#for-all) BotAction can have any loop iteration aborted without aborting the loop itself. A default AbortLine Signal will abort one loop iteration, while AbortLine Signals with `assembledLines` two or more will abort the entire loop. -BotAction's with unique aborting behavior have their aborting functionality explained in their documentation, listed under API in the sidebar. \ No newline at end of file +All BotActions with unique aborting behavior have their aborting functionality explained in their respective documentation pages. \ No newline at end of file