-
Notifications
You must be signed in to change notification settings - Fork 4
/
index.html
595 lines (572 loc) · 26.1 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
<!DOCTYPE html>
<html>
<head>
<title>Bypass Cors | Adding Horsepower to Web Apps!</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script src='https://unpkg.com/vue@2'></script>
<link rel="shortcut icon" href="github_pages/favicon.ico" type="image/x-icon">
<link rel="icon" href="github_pages/favicon.ico" type="image/x-icon">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.5.0/themes/prism.min.css">
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css">
<link rel="stylesheet" href="github_pages/index.css">
<link rel="stylesheet" href="github_pages/loading.css">
<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
</head>
<body>
<div id='app'>
<div class="container-fluid">
<transition name='modal' v-if="isLoading">
<div class="modal-mask">
<div class="modal-container text-center">
<img src="github_pages/loading.gif" alt="">
<p class="status saving">
{{status}}
<span>.</span>
<span>.</span>
<span>.</span>
</p>
</div>
</div>
</transition>
<div class="github">
<a href="https://github.com/chrishham/BypassCors" target="_blank"><img width="149" height="149"
src="https://github.blog/wp-content/uploads/2008/12/forkme_right_darkblue_121621.png?resize=149%2C149"
class="attachment-full size-full" alt="Fork me on GitHub" data-recalc-dims="1"></a>
</div>
<div class="row">
<div class="col-md-2 col-sm-12" id="right-navbar">
<ul class="nav flex-column">
<li class="nav-item">
<a class="nav-link" href="#introduction">Introduction</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="#installation">Installation</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="#how_to_use">How to use</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#instructions_for_developers">Instructions for Developers</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#api">API</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#demo">Demo</a>
</li>
</ul>
</div>
<div class="offset-md-2 col-md-9 offset-lg-3 col-lg-7">
<header class="text-center">
<h1>Bypass Cors
<img src="github_pages/[email protected]" alt="Bypass CORS logo">
</h1>
<p>Adding Horsepower to Web Apps!</p>
</header>
<hr>
<h2 id="introduction" class="text-center">Introduction</h2>
<img src="github_pages/Bypass_CORS.png" class="img-fluid" alt="Bypass Cors Explained">
<p> NEVER again let not enabled CORS domains , stand in your Web App's way to acquiring
publicly available web resources !
</p>
<h4>
What it does
</h4>
<p>
It helps Web Apps running at user's Browser , or any program that speaks HTTP ,
to fetch HTML files residing anywhere in the web , given that the user's browser can navigate to.
</p>
<h4>The problem that it solves</h4>
<p>
All modern Web Browsers enforce MANY restrictions to the way a Web App , which services the user,
makes use of the web.
</p>
One of them is the <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS">Cross-Origin Resource
Sharing (CORS)</a> mechanism.
</p>
<p>
In simple terms , CORS mechanism doesnt't allow ANY Web App to access a resource of another Domain,
unless the other Domain has EXPLICITLY allowed to.
</p>
<p>
That means that despite that you , the user , have access to all the content of the web
(given that you are not behind a firewall ) , the Web App that needs it to provide you with
updated /currated /aggregated content from other resources , DOESN'T!
</p>
<p>
For example , if a Web App needs to display the list of all the "World Chess Champions" and issues a GET
request to "https://en.wikipedia.org/wiki/World_Chess_Championship", it will be greeted with this dreadful
error message:
<br>
<img src="github_pages/CORS_error.png" alt="CORS error" class="cors_error mt-2 img-fluid">
</p>
<p>
Bypass Cors does what it's name says: Bypasses CORS restrictions PERMANENTLY!
</p>
<p>
Watch how this Web App (that you are viewing right now) , manages to get the HTML of the "World Chess
Champions" ,
at the <a href="#demo">demo</a> section.
</p>
<h4>How it works</h4>
<p>
Bypass Cors is an installable <a href="https://electronjs.org/">Electron</a> App (Windows/Mac/Linux).
</p>
<p>
When Bypass Cors launches, it checks the status of the machine's Internet Connection, by issuing
an Http GET request to <i>"https://www.google.com"</i>.
</p>
<p>
If the machine hasn't access to the Internet, a red light and a warning message appears ,
telling the user to check his proxy settings / internet connection.
</p>
<p>
If everything is ok , a green light goes on. An <a href="https://expressjs.com/" target="_blank">Express</a>
Web Server begins listening for requests at "<i>http://localhost:3167/</i>" (user can change the port) .
</p>
<p>
The Web App can issue a <a href="#instructions_for_developers">POST request</a> to this endpoint and take as
a response the HTML
document it wants.
</p>
<p>
The Web App can then parse the retreived HTML and extract any data it needs from it, using any HTML parser
it wants.
</p>
<h4>Highlights</h4>
<ul>
<li><a href="https://github.com/chrishham/BypassCors" target="_blank">Open Source!</a></li>
<li>Free of Charge!</li>
<li>Full page render & Javascript support!</li>
<li>Cookie support!</li>
<li>POST request support!</li>
<li>Ideal for Real Time Scraping!</li>
</ul>
<hr>
<h2 id="installation" class="text-center">Installation</h2>
<p class="text-left">
<b>Step 1:</b>
Download the appropriate installer for your machine:
</p>
<section class="text-center">
<button class="btn btn-primary mb-4">
<a :href="latestReleaseBaseUrl + windowsInstaller">
<img class="logo" src="github_pages/WindowsLogo.png" alt="Windows Logo"> Windows
</a>
</button>
<button class="btn btn-info mb-4 ml-4 ">
<a :href="latestReleaseBaseUrl + macInstaller">
<img class="logo" src="github_pages/MacLogo.png" alt="Mac Logo"> Mac
</a>
</button>
<button class="btn btn-success mb-4 ml-4">
<a :href="latestReleaseBaseUrl + linuxInstaller">
<img class="logo linux" src="github_pages/LinuxLogo.png" alt="Linux Logo"> Linux
</a>
</button>
</section>
<p class="text-left">
<b>Step 2:</b>
Follow the instructions specific to your os :
</p>
<ul class="nav nav-tabs">
<li class="nav-item">
<a class="nav-link" href="#" @click.prevent="osActive='Windows'"
:class="{ active: osActive==='Windows' }">Windows</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#" @click.prevent="osActive='Mac'" :class="{ active: osActive==='Mac' }">Mac</a>
</li>
<li class="nav-item">
<a class="nav-link" href="#" @click.prevent="osActive='Linux'"
:class="{ active: osActive==='Linux' }">Linux</a>
</li>
</ul>
<div class="text-left installation">
<section v-if="osActive==='Windows'">
<p class="mt-3"><b> 1.</b>Open the folder where <b> {{windowsInstaller}} </b> was saved. <em> Don't try to
run the installer directly from within your browser downloads section; it won't work.</em>
Double-click the file.</p>
<p class="mt-3"><b> 2.</b>You may run into the following warning message, initiated from
<b>"Windows Defender Smartscreen" </b>: </p>
<img src="github_pages/WindowsDefender1.png" class="img-fluid" alt="WindowsDefenderFirstScreen">
<p class="mt-2">Click at <b>"More info" </b>.</p>
<p> When the warning message expands, click the <b>“Run anyway” </b> button</p>
<p class="mt-3"><b> 3.</b>When installation completes , the <b>"Windows Defender Firewall"</b> will ask
you to grant access to private and public networks. <u> Tick both checkboxes.</u></p>
</section>
<section v-if="osActive==='Mac'">
<p class="mt-3"><b> 1.</b>Double-click the downloaded <b> {{macInstaller}} </b>.</p>
<p class="mt-3"><b> 2.</b>Drag and drop the Bypass Cors icon to the Application's folder.</p>
<img src="github_pages/MacInstall.png" class="macInstall img-fluid" alt="MacInstall">
<p class="mt-3"><b> 3.</b>When you launch the app for the first time, you may get the message:
"You cannot open an app from an unidentified developer" . To proceed, locate the installed Bypass Cors
app in the Finder (under Applications Folder),
right-click it and select "Open". More detailed instructions can be found <a
href="https://support.apple.com/kb/ph25088?locale=en_US" target="_blank"> here</a>.
</p>
</section>
<section v-if="osActive==='Linux'">
<p class="mt-3"><b> 1.</b>
Right-Click the downloaded <b> {{linuxInstaller}} </b> => Properties => Permissions => Execute => Allow
executing file as program =>
<u>tick the checkbox</u>.</p>
<p class="mt-3"><b> 2.</b>Double-click the installer. You can choose to run as a standalone( no
installation).</p>
</section>
</div>
<hr>
<h2 id="how_to_use" class="text-center"> How to use</h2>
<figure class="text-center">
<img src="github_pages/app_main_photo.png" alt="Bypass Cors app overview"
class="img-fluid app_main_photo mx-auto d-block">
<figcaption>Bypass Cors window.</figcaption>
</figure>
<p class="mt-2">
When the app launches it checks the status of the Internet connectivity ,
by trying to access <i>"https://www.google.com"</i>. If everything is ok, you get
the green light like the image above.
</p>
<p>
In order for a Web App to be able to communicate with Bypass Cors,
you have to :
<ol>
<li>Add it's domain to the "Whitelist Domains" textarea, e.g. "github.io".</li>
<li>Press the Restart Server button for the change to take effect.</li>
</ol>
</p>
<hr>
<h2 id="instructions_for_developers" class="text-center">Instructions for Developers</h2>
<p><b>Three steps:</b></p>
<ol>
<li>Prod your users to install Bypass Cors to their machines.</li>
<li>Tell them to add your domain to Bypass Cors's '"Whitelist Domains", e.g. "example.com".
For the change to take effect , they have to press the "Restart Server" button.
</li>
<li>Issue an http POST request from your web app, to the url Bypass Cors is listening at.
Default address is http://localhost:3167/, but user can change the port. Example below makes
use of the ES6 feature : <a href="http://es6-features.org/#PropertyShorthand" target="_blank"> property
shorthand.</a> Look up what
each variable does , at the <a href="#api">API </a> section.
<pre><code class="language-javascript"> let bypassCorsUrl = "http://localhost:3167/"
let url = "https://en.wikipedia.org/wiki/World_Chess_Championship"
let cookies = [ ]
let post = null
let fullPageRender = false
let javascript = ""
let scrollInterval = 500
let debug = false
axios.post(bypassCorsUrl,{
url,headers,cookies,post,fullPageRender,javascript,scrollInterval,debug})
.then(res => {
let { html, cookies ,fullResponse } = res.data
// rest of your code here
})
</code></pre>
</li>
</ol>
<hr>
<h2 class="text-center" id="api">API</h2>
<pre><code class="language-javascript">axios.post(
bypassCorsUrl, // String : REQUIRED.
// The url Bypass Cors is listening at.
// e.g. http://localhost:3167/
{
url, // String : REQUIRED
// The target url.
// e.g. https://en.wikipedia.org/wiki/World_Chess_Championship
headers, // Object : REQUIRED.
// The headers which will be used for the final
// GET/POST request to the target url.
// e.g.
// {
// "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
// "Accept-language": "en-GB,en;q=0.9,en-US;q=0.8,el;q=0.7",
// "Accept-Encoding": "gzip,deflate",
// "cache-control": "max-age=0",
// "Host": "en.wikipedia.org",
// "Origin": "https://en.wikipedia.org",
// "upgrade-insecure-requests": "1",
// "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"
// }
cookies, // Array of Objects : OPTIONAL
// The cookies to be sent with the GET/POST request.
// The cookie Object can have the following pairs of keys :
// "key"/"value" or "name"/"value".
// e.g.
// [
// {
// "key": "cookie_key",
// "value": "cookie_value"
// },
// {
// "name": "cookie_key",
// "value": "cookie_value"
// },
// ....
// ]
post, // Object : OPTIONAL , defaults to null.
// FormData Object to be sent along post request
// If it is !== null a POST request will be performed to target Url ,
// else GET request will be performed.
// e.g. { number_of_items:6 }
fullPageRender, // Boolean : OPTIONAL , defaults to false.
// Cant't be combined with post : true.
// If you want to get html source after
// full page rendering, i.e. all javascript
// has been executed and all resources fetched ,
// set this to true.
// e.g. true
javascript, // String : OPTIONAL , defaults to ""
// Applied only when fullPageRender : true.
// Evals and executes the given Javascript code
// when page is fully loaded.
// Usefull when user action need's to be taken,
// before Bypass Cors retreives html to web app.
// e.g.
// `
// let button = document.querySelector('.show-more-items');
// button.click();
// `
scrollInterval, // Number : OPTIONAL , defaults to 500.
// Applied only when fullPageRender : true.
// After custom Javascript execution (if any),
// Bypass CORS scrolls down the page from top
// to bottom , at "scrollInterval" ms interval,
// in order to retrieve the real innerHTML.
// e.g. 1000
debug // Boolean : OPTIONAL , defaults to false
// Applied only when fullPageRender : true AND
// your web app is running at localhost (TEST ONLY).
// Shows the Electron Browser Window loading the page.
// You need to manually close the window,
// after execution.
// e.g. true
})
.then(res => { // here you get the requested html + cookies + fullResponse(only when fullPageRender=false)
let { html, cookies , fullResponse } = res.data
}) </code></pre>
<hr>
<div id="demo">
<h2 class="text-center"> Demo</h2>
<p> To see Bypass Cors in action, you have to :</p>
<ol>
<li>Install Bypass Cors to your machine.</li>
<li>Add the domain "github.io" to
the "Whitelist Domains" textarea.
</li>
<li>Press the "Restart Server" button.</li>
</ol>
<section class="settings">
<h3 class="text-center">Settings Area</h3>
<div class="form-group">
<label for="bypassCorsUrl"><b>Bypass Cors Url :</b> </label>
<input type="text" v-model="bypassCorsUrl" id="bypassCorsUrl">
</div>
<div class="form-group">
<label for="targetUrl"><b>Target Url : </b> </label>
<input type="text" v-model="url" id="targetUrl" class="targetUrl">
</div>
<div class="form-group">
<label for="headersTxt"><b> Headers : </b> </label>
<textarea v-model="headersTxt" id="headersTxt" class="headersTxt">
</textarea>
</div>
<div class="cookies">
<label for="cookiesTxt"><b> Cookies : </b> </label>
<textarea v-model="cookiesTxt" id="cookiesTxt" class="cookiesTxt">
</textarea>
</div>
<div class="form-group">
<label for="post"><b>Post : </b> </label>
<label for="postYes"> Yes </label>
<input type="radio" v-model="post" id="postYes" :value="true">
<label for="postNo"> No </label>
<input type="radio" v-model="post" id="postNo" :value="false">
</div>
<div class="form-group">
<label for="fullPageRender"><b>Full Page Render : </b> </label>
<label for="fullPageRenderYes"> Yes </label>
<input type="radio" v-model="fullPageRender" id="fullPageRenderYes" :value="true">
<label for="fullPageRenderNo"> No </label>
<input type="radio" v-model="fullPageRender" id="fullPageRenderNo" :value="false">
</div>
<div class="javascript">
<label for="javascriptTxt"><b> Javascript : </b> </label>
<textarea v-model="javascriptTxt" id="javascriptTxt" class="javascriptTxt">
</textarea>
</div>
<div class="form-group">
<label for="scrollInterval"><b>Scroll Interval : </b> </label>
<input type="number" min="0" max="10000" v-model="scrollInterval" id="scrollInterval"
class="scrollInterval"> ms
</div>
<div class="form-group">
<label for="debug"><b>Debug : </b> </label>
<label for="debugYes"> Yes </label>
<input type="radio" v-model="debug" id="debugYes" :value="true">
<label for="debugNo"> No </label>
<input type="radio" v-model="debug" id="debugNo" :value="false">
</div>
<div class="text-center">
<button class="btn btn-primary" @click="getHTML"> Get HTML!</button>
<span class="ml-3">Execution time : {{executionSeconds}}
second{{executionSeconds===1 ? "" : 's'}}.</span>
</div>
</section>
<section class="html mt-3">
<h3 class="text-center">HTML rendered in an iframe</h3>
<iframe ref="iframeContent" src="about:blank" class="whatScraperSees"></iframe>
</section>
</div>
<hr>
<p class="text-center"> An
<a href="https://electronjs.org/" target="_blank"> Electron </a>
app, created with ❤️ by Christopher Chamaletsos.
<br>
<a href="https://github.com/chrishham/BypassCors/blob/master/LICENSE" target="_blank">MIT</a> license.
Hosted at <a href="https://github.com/chrishham/BypassCors" target="_blank"><img
src="github_pages/GithubIcon.svg" alt="Github Icon" class="githubIcon"> </a>.
</p>
</div> <!-- MAIN CONTENT -->
</div>
</div>
</div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.5.0/prism.min.js"></script>
<script>
var app = new Vue({
el: '#app',
data: {
isLoading: false,
status: "",
osActive: "Windows",
version: '',
bypassCorsUrl: 'http://localhost:3167/',
url: 'https://en.wikipedia.org/wiki/World_Chess_Championship',
headersTxt: `{
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-language": "en-GB,en;q=0.9,en-US;q=0.8,el;q=0.7",
"Accept-Encoding": "gzip,deflate",
"cache-control": "max-age=0",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"
}`,
cookiesTxt: '',
post: false,
fullPageRender: false,
javascriptTxt: "",
scrollInterval: 500,
debug: false,
html: "",
executionSeconds: 0
},
computed: {
latestReleaseBaseUrl() {
return `https://github.com/chrishham/BypassCors/releases/download/v${this.version}/`
},
windowsInstaller() {
return `bypass-cors-setup-${this.version}.exe`
},
macInstaller() {
return `bypass-cors-${this.version}.dmg`
},
linuxInstaller() {
return `bypass-cors-${this.version}-x86_64.AppImage`
}
},
methods: {
getHTML() {
let headers
let cookies
try {
headers = JSON.parse(this.headersTxt)
} catch (e) {
return alert(e)
}
try {
if (this.cookiesTxt) {
cookies = JSON.parse(this.cookiesTxt)
}
} catch (e) {
console.log(e)
}
this.html = ""
this.isLoading = true
this.status = "Fetching requested html"
let startTime = new Date();
axios.post(
this.bypassCorsUrl,
{
url: this.url,
headers,
cookies,
post: this.post,
fullPageRender: this.fullPageRender,
javascript: this.javascript,
scrollInterval: +this.scrollInterval,
debug: this.debug
})
.then(res => { // here you get the requested html + cookies!
let { html, cookies } = res.data
this.cookiesTxt = JSON.stringify(cookies)
this.html = html
let frame = this.$refs.iframeContent.contentWindow;
frame.document.open("text/html", "replace");
frame.document.write(html);
frame.document.close();
})
.catch(error => {
alert(error)
console.log(error)
})
.then(() => {
this.isLoading = false
this.status = ""
let endTime = new Date();
let timeDiff = endTime - startTime; //in ms
// strip the ms
timeDiff /= 1000;
// get seconds
let seconds = Math.round(timeDiff);
this.executionSeconds = seconds;
})
}
},
created() {
let os = getOS()
if (os !== "Mac" && os !== "Linux") os = 'Windows'
this.osActive = os
axios('https://api.github.com/repos/chrishham/BypassCors/releases/latest')
.then(result => {
console.log(result.data.tag_name)
this.version = result.data.tag_name.slice(1)
})
.catch(error => {
console.log(error)
this.version = '0.1.0'
})
}
})
function getOS() {
var userAgent = window.navigator.userAgent,
platform = window.navigator.platform,
macosPlatforms = ['Macintosh', 'MacIntel', 'MacPPC', 'Mac68K'],
windowsPlatforms = ['Win32', 'Win64', 'Windows', 'WinCE'],
iosPlatforms = ['iPhone', 'iPad', 'iPod'],
os = null;
if (macosPlatforms.indexOf(platform) !== -1) {
os = 'Mac';
} else if (iosPlatforms.indexOf(platform) !== -1) {
os = 'iOS';
} else if (windowsPlatforms.indexOf(platform) !== -1) {
os = 'Windows';
} else if (/Android/.test(userAgent)) {
os = 'Android';
} else if (!os && /Linux/.test(platform)) {
os = 'Linux';
}
return os;
}
</script>
</body>
</html>