-
Notifications
You must be signed in to change notification settings - Fork 31
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Pull data files from ICU GitHub instead of npm #53
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
module.exports = { | ||
env: { | ||
commonjs: true, | ||
es6: true, | ||
node: true | ||
}, | ||
extends: 'standard', | ||
globals: { | ||
Atomics: 'readonly', | ||
SharedArrayBuffer: 'readonly' | ||
}, | ||
parserOptions: { | ||
ecmaVersion: 2018 | ||
}, | ||
rules: { | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
name: Integration Test npm | ||
|
||
on: [push, pull_request] | ||
|
||
jobs: | ||
build: | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
container: | ||
- node:12 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. May be worth expanding out the matrix to cover supported versions or at least a version with NPM 6 vs NPM 7 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @nschonni makese sense (although probably good for a separate PR), do you have a recommendation? |
||
- node:12-slim | ||
# Will fail on versions that aren't in icu4c-data | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v2 | ||
- uses: actions/checkout@v2 | ||
with: | ||
repository: nodejs/full-icu-test | ||
path: full-icu-test | ||
- name: Install full-icu | ||
run: docker run --rm -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} env FULL_ICU_PREFER_NPM=1 npm i --no-package-lock --unsafe-perm | ||
- name: Test full-icu | ||
run: docker run --rm -e NODE_ICU_DATA=. -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} node ./full-icu-test/test.js |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,3 +2,5 @@ node_modules | |
*.dat | ||
npm-debug.log | ||
/yarn.lock | ||
package-lock.json | ||
/.nyc_output |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,4 +2,5 @@ | |
node_modules | ||
.svn | ||
.git | ||
npm-debug.log | ||
npm-debug.log | ||
/test |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
language: node_js | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would probably drop this, since you've got this running on Actions now |
||
node_js: | ||
- '8' | ||
- '10' | ||
- '11' | ||
- '12' | ||
script: | ||
- npm install | ||
- npm t | ||
cache: | ||
directories: | ||
- node_modules | ||
- ".nvm" | ||
# this is a comment |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure if this is right, or just got copied across from some of the other scripts There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it's copied from install-spawn |
||
|
||
// Install by fetching ICU source tarball | ||
// This will only work for little endian systems, but will work for ancient ICU (back to v50) | ||
|
||
const fs = require('fs') | ||
const { URL } = require('url') | ||
const process = require('process') | ||
const myFetch = require('./myFetch') | ||
const yauzl = require('yauzl') | ||
|
||
module.exports = async function installFromGithub (fullIcu, advice) { | ||
const { icudat, icuend } = fullIcu | ||
if (fs.existsSync(icudat)) { | ||
console.log(` √ ${icudat} (exists)`) | ||
return | ||
} | ||
|
||
// Example URL: | ||
// https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip | ||
const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/' | ||
const baseUrl = new URL(_baseUrl) | ||
const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-') | ||
// ICU v67/v68 use "68.1" and "67.1" in the filename instead of 68_1 and 69_1 | ||
// https://unicode-org.atlassian.net/browse/ICU-21764 | ||
// Can remove this conditional if the files are updated later. | ||
const versionsAsUnderscore = (fullIcu.icumaj >= 69) ? fullIcu.icuver.replace(/\./g, '_') : fullIcu.icuver | ||
const tag = `release-${versionsAsHyphen}` | ||
const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip` | ||
const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl) | ||
console.log(fullUrl.toString()) | ||
const [srcZip, tmpd] = await myFetch(fullUrl) | ||
|
||
console.log(srcZip, tmpd) | ||
|
||
// now, unpack it | ||
console.log(`Looking for ${icudat}`) | ||
return new Promise((resolve, reject) => | ||
yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => { | ||
if (err) return reject(err) | ||
zipfile.readEntry() | ||
zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`))) | ||
zipfile.on('entry', (entry) => { | ||
if (entry.fileName.endsWith('/')) { | ||
zipfile.readEntry() | ||
} else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) { | ||
console.log('found ' + entry.fileName) | ||
zipfile.openReadStream(entry, (err, readStream) => { | ||
if (err) return reject(err) | ||
readStream.on('end', () => zipfile.readEntry()) | ||
const pipeOut = fs.createWriteStream(icudat) | ||
readStream.pipe(pipeOut) | ||
console.log(` √ ${icudat} (from ICU binary data tarball)`) | ||
return resolve() | ||
}) | ||
} else { | ||
zipfile.readEntry() | ||
} | ||
}) | ||
})) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved. | ||
|
||
// Install by fetching ICU source tarball | ||
// This will only work for little endian systems, but will work for ancient ICU (back to v50) | ||
|
||
const fs = require('fs') | ||
const { URL } = require('url') | ||
const process = require('process') | ||
const myFetch = require('./myFetch') | ||
const yauzl = require('yauzl') | ||
|
||
module.exports = async function installFromGithub (fullIcu, advice) { | ||
const { icudat, icuend } = fullIcu | ||
|
||
if (fs.existsSync(icudat)) { | ||
console.log(` √ ${icudat} (exists)`) | ||
return | ||
} | ||
|
||
if (icuend !== 'l') { | ||
// Should not hit this, as versions 67 and prior are already in NPM | ||
console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.') | ||
} | ||
|
||
// var args; | ||
// https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip | ||
const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/' | ||
const baseUrl = new URL(_baseUrl) | ||
const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-') | ||
const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_') | ||
const tag = `release-${versionsAsHyphen}` | ||
const file = `icu4c-${versionsAsUnderscore}-src.zip` | ||
const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl) | ||
console.log(fullUrl.toString()) | ||
const [srcZip, tmpd] = await myFetch(fullUrl) | ||
|
||
console.log(srcZip, tmpd) | ||
|
||
// now, unpack it | ||
console.log(`Looking for ${icudat}`) | ||
return new Promise((resolve, reject) => | ||
yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => { | ||
if (err) return reject(err) | ||
zipfile.readEntry() | ||
zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`))) | ||
zipfile.on('entry', (entry) => { | ||
if (entry.fileName.endsWith('/')) { | ||
zipfile.readEntry() | ||
} else if (entry.fileName.endsWith('/' + icudat)) { | ||
console.log('found ' + entry.fileName) | ||
zipfile.openReadStream(entry, (err, readStream) => { | ||
if (err) return reject(err) | ||
readStream.on('end', () => zipfile.readEntry()) | ||
const pipeOut = fs.createWriteStream(icudat) | ||
readStream.pipe(pipeOut) | ||
console.log(` √ ${icudat} (from ICU source tarball)`) | ||
return resolve() | ||
}) | ||
} else { | ||
zipfile.readEntry() | ||
} | ||
}) | ||
})) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved. | ||
|
||
const os = require('os') | ||
const path = require('path') | ||
const fs = require('fs') | ||
const { sep } = require('path') | ||
|
||
function getFetcher (u) { | ||
if (u.protocol === 'https:') return require('https') | ||
if (u.protocol === 'http:') return require('http') | ||
return null | ||
} | ||
|
||
/** | ||
* @param {URL} fullUrl url to fetch | ||
* @returns {Promse<String[]>} filename, tmpdir | ||
*/ | ||
function myFetch (fullUrl) { | ||
return new Promise((resolve, reject) => { | ||
const fetcher = getFetcher(fullUrl) | ||
console.log('Fetch:', fullUrl.toString()) | ||
if (!fetcher) { | ||
return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`)) | ||
} | ||
|
||
fetcher.get(fullUrl, res => { | ||
const length = res.headers['content-length'] | ||
if (res.statusCode === 302 && res.headers.location) { | ||
return resolve(myFetch(new URL(res.headers.location))) | ||
} else if (res.statusCode !== 200) { | ||
return reject(Error(`Bad status code ${res.statusCode}`)) | ||
} | ||
const tmpd = fs.mkdtempSync(`${os.tmpdir()}${sep}`) | ||
const tmpf = path.join(tmpd, 'icu-download.zip') | ||
let gotSoFar = 0 | ||
console.dir(tmpd) | ||
|
||
res.on('data', data => { | ||
gotSoFar += data.length | ||
fs.appendFileSync(tmpf, data) | ||
// console.dir(res.headers); | ||
process.stdout.write(`${gotSoFar}/${length}\r`) | ||
// console.log(`chunk: ${data.length}`); | ||
}) | ||
res.on('end', () => { | ||
resolve([tmpf, tmpd]) | ||
console.log(`${gotSoFar}/${length}\n`) | ||
}) | ||
res.on('error', error => { | ||
fs.unlinkSync(tmpf) | ||
fs.rmdirSync(tmpd) | ||
console.error(error) | ||
return reject(error) | ||
}) | ||
}) | ||
}) | ||
} | ||
|
||
module.exports = myFetch |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,11 @@ | ||
#!/usr/bin/env node | ||
const { spawn } = require('child_process') | ||
const data = require('./full-icu') | ||
const env = data.icu_small ? { | ||
...process.env, | ||
NODE_ICU_DATA: data.datPath() | ||
} : process.env | ||
const env = data.icu_small | ||
? { | ||
...process.env, | ||
NODE_ICU_DATA: data.datPath() | ||
} | ||
: process.env | ||
|
||
spawn('/usr/bin/env', ['node', ...process.argv.slice(2)], { env, stdio: 'inherit' }) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,11 @@ | ||
{ | ||
"name": "full-icu", | ||
"version": "1.3.5-0", | ||
"version": "1.4.0-0", | ||
"description": "install 'full-icu' data for your current node", | ||
"scripts": { | ||
"lint": "standard", | ||
"postinstall": "node postinstall.js" | ||
"lint": "standard && eslint *.js test/*.js", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Think you can skip the call to |
||
"postinstall": "node postinstall.js", | ||
"test": "tap test/*.js" | ||
}, | ||
"keywords": [ | ||
"icu4c" | ||
|
@@ -23,7 +24,17 @@ | |
"bugs": { | ||
"url": "https://github.com/unicode-org/full-icu-npm/issues" | ||
}, | ||
"dependencies": { | ||
"yauzl": "^2.10.0" | ||
}, | ||
"devDependencies": { | ||
"standard": "^16.0.3" | ||
"eslint": "^7.7.0", | ||
"eslint-config-standard": "^16.0.3", | ||
"eslint-plugin-header": "^3.0.0", | ||
"eslint-plugin-import": "^2.24.2", | ||
"eslint-plugin-node": "^11.1.0", | ||
"eslint-plugin-promise": "^4.2.1", | ||
"standard": "^16.0.3", | ||
"tap": "^15.0.10" | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
v2 also allows native caching https://github.com/actions/setup-node#caching-packages-dependencies if you set it. caching probably isn't needed right now, since there are no lock files