diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..f3245e76 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +end_of_line = lf +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..30262b05 --- /dev/null +++ b/.env.example @@ -0,0 +1,40 @@ +# Unused atm (jest sets this to `test`) +NODE_ENV=development + +# Debug node modules - https://nodejs.org/api/cli.html#node_debugmodule +# NODE_DEBUG= + +# Debug node native modules - https://nodejs.org/api/cli.html#node_debug_nativemodule +# NODE_DEBUG_NATIVE= + +# Path to PK executable to override tests/bin target +# PK_TEST_COMMAND= + +# If set, indicates that `PK_TEST_COMMAND` is targetting docker +# PK_TEST_COMMAND_DOCKER= +# Accessing AWS for testnet.polykey.io and mainnet.polykey.io deployment +AWS_DEFAULT_REGION='ap-southeast-2' +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# Path to container registry authentication file used by `skopeo` +# The file has the same contents as `DOCKER_AUTH_CONFIG` +# Use this command to acquire the auth file at `./tmp/auth.json`: +# ``` +# printf 'PASSWORD' | skopeo login \ +# --username 'USERNAME' \ +# --password-stdin \ +# $CI_REGISTRY_IMAGE \ +# --authfile=./tmp/auth.json +# ``` +# REGISTRY_AUTH_FILE= + +# Authenticate to GitHub with `gh` +# GITHUB_TOKEN= + +# To allow testing different executables in the bin tests +# Both PK_TEST_COMMAND and PK_TEST_PLATFORM must be set at the same time +# PK_TEST_COMMAND= #Specify the shell command we want to test against +# PK_TEST_PLATFORM=docker #Overrides the auto set `testPlatform` variable used for enabling platform specific tests +# PK_TEST_TMPDIR= #Sets the `global.tmpDir` variable to allow overriding the temp directory used for tests + diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..72f98eb5 --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +src/proto/* diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 00000000..1eec3982 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,177 @@ +{ + "env": { + "browser": true, + "commonjs": true, + "es2021": true, + "node": true, + "jest": true + }, + "parser": "@typescript-eslint/parser", + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:prettier/recommended" + ], + "plugins": [ + "import" + ], + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "rules": { + "linebreak-style": ["error", "unix"], + "no-empty": 1, + "no-useless-catch": 1, + "no-prototype-builtins": 1, + "no-constant-condition": 0, + "no-useless-escape": 0, + "no-console": "error", + "no-restricted-globals": [ + "error", + { + "name": "global", + "message": "Use `globalThis` instead" + }, + { + "name": "window", + "message": "Use `globalThis` instead" + } + ], + "require-yield": 0, + "eqeqeq": ["error", "smart"], + "spaced-comment": [ + "warn", + "always", + { + "line": { + "exceptions": ["-"] + }, + "block": { + "exceptions": ["*"] + }, + "markers": ["/"] + } + ], + "capitalized-comments": [ + "warn", + "always", + { + "ignoreInlineComments": true, + "ignoreConsecutiveComments": true + } + ], + "curly": [ + "error", + "multi-line", + "consistent" + ], + "import/order": [ + "error", + { + "groups": [ + "type", + "builtin", + "external", + "internal", + "index", + "sibling", + "parent", + "object" + ], + "pathGroups": [ + { + "pattern": "@", + "group": "internal" + }, + { + "pattern": "@/**", + "group": "internal" + } + ], + "pathGroupsExcludedImportTypes": [ + "type" + ], + "newlines-between": "never" + } + ], + "@typescript-eslint/no-namespace": 0, + "@typescript-eslint/no-explicit-any": 0, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-unused-vars": [ + "warn", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_" + } + ], + "@typescript-eslint/no-inferrable-types": 0, + "@typescript-eslint/no-non-null-assertion": 0, + "@typescript-eslint/no-this-alias": 0, + "@typescript-eslint/no-var-requires": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/consistent-type-imports": ["error"], + "@typescript-eslint/consistent-type-exports": ["error"], + "no-throw-literal": "off", + "@typescript-eslint/no-throw-literal": "off", + "@typescript-eslint/no-floating-promises": ["error", { + "ignoreVoid": true, + "ignoreIIFE": true + }], + "@typescript-eslint/no-misused-promises": ["error", { + "checksVoidReturn": false + }], + "@typescript-eslint/await-thenable": ["error"], + "@typescript-eslint/naming-convention": [ + "error", + { + "selector": "default", + "format": ["camelCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "function", + "format": ["camelCase", "PascalCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "variable", + "format": ["camelCase", "UPPER_CASE", "PascalCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "parameter", + "format": ["camelCase"], + "leadingUnderscore": "allow", + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "typeLike", + "format": ["PascalCase"], + "trailingUnderscore": "allowSingleOrDouble" + }, + { + "selector": "enumMember", + "format": ["PascalCase", "UPPER_CASE"] + }, + { + "selector": "objectLiteralProperty", + "format": null + }, + { + "selector": "typeProperty", + "format": null + } + ], + "@typescript-eslint/ban-ts-comment": [ + "error", + { + "ts-ignore": "allow-with-description" + } + ] + } +} diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 00000000..80f58e63 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,23 @@ +# This workflow was added by CodeSee. Learn more at https://codesee.io/ +# This is v2.0 of this workflow file +on: + push: + branches: + - staging + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee + +permissions: read-all + +jobs: + codesee: + runs-on: ubuntu-latest + continue-on-error: true + name: Analyze the repo with CodeSee + steps: + - uses: Codesee-io/codesee-action@v2 + with: + codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + codesee-url: https://app.codesee.io diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..78a1b31b --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +/tmp +/dist +.env* +!.env.example +# nix +/result* +/builds +# node-gyp +/build +# prebuildify +/prebuilds + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# editor +.vscode/ +.idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..f5a0d165 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,662 @@ +workflow: + rules: + # Disable merge request pipelines + - if: $CI_MERGE_REQUEST_ID + when: never + - when: always + +variables: + GIT_SUBMODULE_STRATEGY: recursive + GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" + GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" + # Cache .npm + npm_config_cache: "${CI_PROJECT_DIR}/tmp/npm" + # Prefer offline node module installation + npm_config_prefer_offline: "true" + # Homebrew cache only used by macos runner + HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" + +default: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + interruptible: true + before_script: + # Replace this in windows runners that use powershell + # with `mkdir -Force "$CI_PROJECT_DIR/tmp"` + - mkdir -p "$CI_PROJECT_DIR/tmp" + +# Cached directories shared between jobs & pipelines per-branch per-runner +cache: + key: $CI_COMMIT_REF_SLUG + # Preserve cache even if job fails + when: 'always' + paths: + - ./tmp/npm/ + # Homebrew cache is only used by the macos runner + - ./tmp/Homebrew + # Chocolatey cache is only used by the windows runner + - ./tmp/chocolatey/ + # `jest` cache is configured in jest.config.js + - ./tmp/jest/ + +stages: + - check # Linting, unit tests + - build # Cross-platform library compilation, unit tests + - integration # Cross-platform application bundling, integration tests, and pre-release + - release # Cross-platform distribution and deployment + +check:scratch: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm test -- --ci tests/scratch.test.ts; + ' + allow_failure: true + rules: + - when: manual + +check:lint: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm run lint; + npm run lint-shell; + ' + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:nix-dry: + stage: check + needs: [] + script: + - nix-build -v -v --dry-run ./release.nix + rules: + # Runs on feature and staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^(?:feature.*|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:test-generate: + stage: check + needs: [] + script: + - > + nix-shell --arg ci true --run $' + ./scripts/check-test-generate.sh > ./tmp/check-test.yml; + ' + artifacts: + when: always + paths: + - ./tmp/check-test.yml + rules: + # Runs on feature commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +check:test: + stage: check + needs: + - check:test-generate + trigger: + include: + - artifact: tmp/check-test.yml + job: check:test-generate + strategy: depend + inherit: + variables: false + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + rules: + # Runs on feature commits and ignores version commits + - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Manually run on commits other than master and staging and ignore version commits + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + when: manual + +build:merge: + stage: build + needs: [] + allow_failure: true + script: + # Required for `gh pr create` + - git remote add upstream "$GH_PROJECT_URL" + - > + nix-shell --arg ci true --run $' + gh pr create \ + --head staging \ + --base master \ + --title "ci: merge staging to master" \ + --body "This is an automatic PR generated by the pipeline CI/CD. This will be automatically fast-forward merged if successful." \ + --assignee "@me" \ + --no-maintainer-edit \ + --repo "$GH_PROJECT_PATH" || true; + printf "Pipeline Attempt on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:dist: + stage: build + needs: [] + script: + - > + nix-shell --arg ci true --run $' + npm run build --verbose; + ' + artifacts: + when: always + paths: + - ./dist + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:platforms-generate: + stage: build + needs: [] + script: + - > + nix-shell --arg ci true --run $' + ./scripts/build-platforms-generate.sh > ./tmp/build-platforms.yml; + ' + artifacts: + when: always + paths: + - ./tmp/build-platforms.yml + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:platforms: + stage: build + needs: + - build:platforms-generate + trigger: + include: + - artifact: tmp/build-platforms.yml + job: build:platforms-generate + strategy: depend + inherit: + variables: false + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +build:prerelease: + stage: build + needs: + - build:dist + - build:platforms + # Don't interrupt publishing job + interruptible: false + script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + - echo 'Publishing library prerelease' + - > + nix-shell --arg ci true --run $' + npm publish --tag prerelease --access public; + ' + after_script: + - rm -f ./.npmrc + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:builds: + stage: integration + needs: + - build:dist + - build:platforms + script: + - mkdir -p ./builds + - > + build_application="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr application \ + )" + - > + nix-store --export $( \ + nix-store --query --requisites "$build_application" \ + ) | gzip > ./builds/js-polykey.closure.gz + # non-nix targets + - > + builds="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr docker \ + --attr package.linux.x64.elf \ + --attr package.windows.x64.exe \ + --attr package.macos.x64.macho \ + --attr package.macos.arm64.macho)" + - cp -r $builds ./builds/ + artifacts: + paths: + - ./builds/ + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:deployment: + stage: integration + needs: + - integration:builds + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion + resource_group: integration:deployment + environment: + name: 'testnet' + deployment_tier: 'staging' + url: 'https://testnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'testnet\' "$CI_REGISTRY_IMAGE"; + ' + - echo 'Deploying ECS service to testnet' + - > + nix-shell --run $' + ./scripts/deploy-service.sh \'polykey-testnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:nix: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + script: + - > + build_application="$( \ + gunzip -c ./builds/js-polykey.closure.gz | \ + nix-store --import | \ + tail -1 \ + )" + - $build_application/polykey + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:docker: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + services: + - docker:20.10.16-dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + FF_NETWORK_PER_BUILD: "true" + PK_TEST_PLATFORM: "docker" + PK_TEST_TMPDIR: "${CI_PROJECT_DIR}/tmp/test" + script: + - docker info + - mkdir $PK_TEST_TMPDIR + - > + nix-shell --arg ci true --run $' + image_and_tag="$(docker load --input ./builds/*docker* | cut -d\' \' -f3)"; + PK_TEST_COMMAND="docker run \$DOCKER_OPTIONS $image_and_tag" npm run test; + ' + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:linux: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + image: ubuntu:latest + script: + - for f in ./builds/*-linux-*; do "$f"; done + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +.integration:windows: + inherit: + default: + - interruptible + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + tags: + - windows + before_script: + - mkdir -Force "$CI_PROJECT_DIR/tmp" + script: + - Get-ChildItem -File ./builds/*-win-* | ForEach {& $_.FullName} + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +.integration:macos: + stage: integration + needs: + - integration:builds + - job: integration:deployment + optional: true + tags: + - saas-macos-medium-m1 + image: macos-12-xcode-14 + script: + - for f in ./builds/*-macos-x64*; do "$f"; done + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +integration:prerelease: + stage: integration + needs: + - integration:builds + - job: build:prerelease + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + # - job: integration:windows + # optional: true + # - job: integration:macos + # optional: true + # Don't interrupt publishing job + interruptible: false + # Requires mutual exclusion + resource_group: integration:prerelease + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + script: + - echo 'Publishing application prerelease' + - > + nix-shell --arg ci true --run $' + if gh release view "$CI_COMMIT_TAG" --repo "$GH_PROJECT_PATH" >/dev/null; then \ + gh release \ + upload "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --clobber \ + --repo "$GH_PROJECT_PATH"; \ + else \ + gh release \ + create "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --prerelease \ + --target staging \ + --repo "$GH_PROJECT_PATH"; \ + fi; + ' + - echo 'Prereleasing container image' + - > + nix-shell --arg ci true --run $' + skopeo login \ + --username "$CI_REGISTRY_USER" \ + --password "$CI_REGISTRY_PASSWORD" \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'testnet\' "$CI_REGISTRY_IMAGE"; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Only runs on tag pipeline where the tag is a prerelease version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+-.*[0-9]+$/ + +integration:merge: + stage: integration + needs: + - build:merge + - job: build:platforms + optional: true + - job: integration:nix + optional: true + - job: integration:docker + optional: true + - job: integration:linux + optional: true + # - job: integration:windows + # optional: true + # - job: integration:macos + # optional: true + # Requires mutual exclusion + resource_group: integration:merge + allow_failure: true + variables: + # Ensure that CI/CD is fetching all commits + # this is necessary to checkout origin/master + # and to also merge origin/staging + GIT_DEPTH: 0 + script: + - > + nix-shell --arg ci true --run $' + printf "Pipeline Succeeded on ${CI_PIPELINE_ID} for ${CI_COMMIT_SHA}\n\n${CI_PIPELINE_URL}" \ + | gh pr comment staging \ + --body-file - \ + --repo "$GH_PROJECT_PATH"; + ' + - git remote add upstream "$GH_PROJECT_URL" + - git checkout origin/master + # Merge up to the current commit (not the latest commit) + - git merge --ff-only "$CI_COMMIT_SHA" + - git push upstream HEAD:master + rules: + # Runs on staging commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + # Runs on tag pipeline where the tag is a prerelease or release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:branch: + stage: release + # Only needs integration:builds from the staging branch pipeline + needs: + - project: $CI_PROJECT_PATH + job: integration:builds + ref: staging + artifacts: true + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:tag) + resource_group: release:deployment + environment: + name: 'mainnet' + deployment_tier: 'production' + url: 'https://mainnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + echo \'Deploying ECS service to mainnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on master commits and ignores version commits + - if: $CI_COMMIT_BRANCH == 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/ + +release:deployment:tag: + stage: release + # Tag pipelines run independently + needs: + - integration:builds + - integration:merge + # Don't interrupt deploying job + interruptible: false + # Requires mutual exclusion (also with release:deployment:branch) + resource_group: release:deployment + environment: + name: 'mainnet' + deployment_tier: 'production' + url: 'https://mainnet.polykey.io' + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + # Override CI_REGISTRY_IMAGE to point to ECR + CI_REGISTRY_IMAGE: '015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + script: + - echo 'Deploying container image to ECR' + - > + nix-shell --arg ci true --run $' + aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + echo \'Deploying ECS service to mainnet\'; + ' + after_script: + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Runs on tag pipeline where the tag is a release version + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + +release:distribution: + stage: release + needs: + - build:dist + - build:platforms + - integration:builds + - integration:merge + - release:deployment:tag + # Don't interrupt publishing job + interruptible: false + # Requires mutual exclusion + resource_group: release:distribution + variables: + REGISTRY_AUTH_FILE: "./tmp/registry-auth-file.json" + script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc + - echo 'Publishing library' + - > + nix-shell --arg ci true --run $' + npm publish --access public; + ' + - echo 'Releasing application builds' + - > + nix-shell --arg ci true --run $' + gh release \ + create "$CI_COMMIT_TAG" \ + builds/*.closure.gz \ + builds/*-docker-* \ + builds/*-linux-* \ + builds/*-win-* \ + builds/*-macos-* \ + --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --notes "" \ + --target master \ + --repo "$GH_PROJECT_PATH"; + ' + - echo 'Releasing container image' + - > + nix-shell --arg ci true --run $' + skopeo login \ + --username "$CI_REGISTRY_USER" \ + --password "$CI_REGISTRY_PASSWORD" \ + --authfile "$REGISTRY_AUTH_FILE" \ + "$CI_REGISTRY_IMAGE"; + image=(./builds/*-docker-*); + ./scripts/deploy-image.sh "${image[0]}" \'mainnet\' "$CI_REGISTRY_IMAGE"; + ' + after_script: + - rm -f ./.npmrc + - rm -f "$REGISTRY_AUTH_FILE" + rules: + # Only runs on tag pipeline where the tag is a release version + # This requires dependencies to also run on tag pipeline + # However version tag comes with a version commit + # Dependencies must not run on the version commit + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000..ea6884b3 --- /dev/null +++ b/.npmignore @@ -0,0 +1,17 @@ +.* +/*.nix +/nix +/tsconfig.json +/tsconfig.build.json +/babel.config.js +/jest.config.js +/nodemon.json +/scripts +/src +/tests +/tmp +/docs +/benches +/build +/builds +/dist/tsbuildinfo diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..7c06da2c --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +# Enables npm link +prefix=~/.npm diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..fa9699b8 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2 +} diff --git a/README.md b/README.md index e55642cc..5aa341c5 100644 --- a/README.md +++ b/README.md @@ -1 +1,289 @@ # Polykey-CLI + +staging:[![pipeline status](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/badges/staging/pipeline.svg)](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/commits/staging) +master:[![pipeline status](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/badges/master/pipeline.svg)](https://gitlab.com/MatrixAI/open-source/Polykey-CLI/commits/master) + +Polykey is an open-source decentralized secrets management and sharing system. It is made for today's decentralized world of people, services and devices. + +* Decentralized Encrypted Storage - No storage of secrets on third parties, secrets are stored on your device and synchronised point-to-point between Polykey nodes. +* Secure Peer-to-Peer Communications - Polykey bootstraps TLS keys by federating trusted social identities (e.g. GitHub). +* Secure Computational Workflows - Share secrets (passwords, keys, tokens and certificates) with people, between teams, and across machine infrastructure. + +
+ +
+ +Polykey synthesizes a unified workflow between interactive password management and infrastructure key management. + +You have complete end-to-end control and privacy over your secrets, with no third-party data collection. + +Polykey runs on distributed keynodes referred to as "nodes". Any computing system can run multiple keynodes. Each node manages one or more vaults which are encrypted filesystems with automatic version history. Vaults can be shared between the nodes. + +This repository is the core library for Polykey. + +The Polykey project is split up into these main repositories: + +* [Polykey](https://github.com/MatrixAI/Polykey) - Polykey Core Library +* [Polykey-CLI](https://github.com/MatrixAI/Polykey-CLI) - CLI of Polykey +* [Polykey-Desktop](https://github.com/MatrixAI/Polykey-Desktop) - Polykey Desktop (Windows, Mac, Linux) application +* [Polykey-Mobile](https://github.com/MatrixAI/Polykey-Mobile) - Polykey Mobile (iOS & Android) Application + +Have a bug or a feature-request? Please submit it the issues of the relevant subproject above. + +For tutorials, how-to guides, reference and theory, see the [docs](https://polykey.io/docs). + +Have a question? Join our [discussion board](https://github.com/MatrixAI/Polykey/discussions). + +Our main website is https://polykey.io + +## Installation + +Note that JavaScript libraries are not packaged in Nix. Only JavaScript applications are. + +Building the package: + +```sh +nix-build -E '(import ./pkgs.nix {}).callPackage ./default.nix {}' +``` + +### Nix/NixOS + +Building the releases: + +```sh +nix-build ./release.nix --attr application +nix-build ./release.nix --attr docker +nix-build ./release.nix --attr package.linux.x64.elf +nix-build ./release.nix --attr package.windows.x64.exe +nix-build ./release.nix --attr package.macos.x64.macho +``` + +Install into Nix user profile: + +```sh +nix-env -f ./release.nix --install --attr application +``` + +### Docker + +Install into Docker: + +```sh +loaded="$(docker load --input "$(nix-build ./release.nix --attr docker)")" +image="$(cut -d' ' -f3 <<< "$loaded")" +docker run -it "$image" +``` + +## Development + +Run `nix-shell`, and once you're inside, you can use: + +```sh +# install (or reinstall packages from package.json) +npm install +# build the dist +npm run build +# run the repl (this allows you to import from ./src) +npm run ts-node +# run the tests +npm run test +# lint the source code +npm run lint +# automatically fix the source +npm run lintfix +``` + +### Calling Commands + +When calling commands in development, use this style: + +```sh +npm run polykey -- p1 p2 p3 +``` + +The `--` is necessary to make `npm` understand that the parameters are for your own executable, and not parameters to `npm`. + +### Using the REPL + +``` +$ npm run ts-node +> import fs from 'fs'; +> fs +> import { Library } from '@'; +> Library +> import Library as Library2 from './src/lib/Library'; +``` + +You can also create test files in `./src`, and run them with `npm run ts-node ./src/test.ts`. + +This allows you to test individual pieces of typescript code, and it makes it easier when doing large scale architecting of TypeScript code. + +### Path Aliases + +Due to https://github.com/microsoft/TypeScript/issues/10866, you cannot use path aliases without a bundler like Webpack to further transform the generated JavaScript code in order to resolve the path aliases. Because this is a simple library demonstration, there's no need to use a bundler. In fact, for such libraries, it is far more efficient to not bundle the code. + +However, we have left the path alias configuration in `tsconfig.json`, `jest.config.js` and in the tests we are making use of the `@` alias. + +### Local Package Linking + +When developing on multiple NPM packages, it can be easier to use `npm link` so that changes are immediately reflected rather than repeatedly publishing packages. To do this, you need to use `npm link`. After linking a local directory, you need to provide `tsconfig.json` paths so TypeScript compiler can find the right files. + +For example when linking `@matrixai/db` located in `../js-db`: + +```sh +npm link ../js-db +``` + +You would need to add these paths to `tsconfig.json`: + +``` + "paths": { + "@": ["index"], + "@/*": ["*"], + "@matrixai/db": ["../node_modules/@matrixai/db/src"], + "@matrixai/db/*": ["../node_modules/@matrixai/db/src/*"] + }, +``` + +### Native Module Toolchain + +There are some nuances when packaging with native modules. +Included native modules are level witch include leveldown and utp-native. + +If a module is not set to public then pkg defaults to including it as bytecode. +To avoid this breaking with the `--no-bytecode` flag we need to add `--public-packages "*"` + +#### leveldown + +To get leveldown to work with pkg we need to include the prebuilds with the executable. +after building with pkg you need to copy from `node_modules/leveldown/prebuilds` -> `path_to_executable/prebuilds` +You only need to include the prebuilds for the arch you are targeting. e.g. for linux-x64 you need `prebuild/linux-x64`. + +The folder structure for the executable should look like this. +- linux_executable_elf +- prebuilds + - linux-x64 + - (node files) + +#### threads.js + +To make sure that the worker threads work properly you need to include the compiled worker scripts as an asset. +This can be fixed by adding the following to `package.json` + +```json +"pkg": { + "assets": "dist/bin/worker.js" + } +``` + +If you need to include multiple assets then add them as an array. + +```json +"pkg": { + "assets": [ + "node_modules/utp-native/**/*", + "dist/bin/worker.js" + ] + } +``` + +### Docs Generation + +```sh +npm run docs +``` + +See the docs at: https://matrixai.github.io/TypeScript-Demo-Lib/ + +### Publishing + +Publishing is handled automatically by the staging pipeline. + +Prerelease: + +```sh +# npm login +npm version prepatch --preid alpha # premajor/preminor/prepatch +git push --follow-tags +``` + +Release: + +```sh +# npm login +npm version patch # major/minor/patch +git push --follow-tags +``` + +Manually: + +```sh +# npm login +npm version patch # major/minor/patch +npm run build +npm publish --access public +git push +git push --tags +``` +### Packaging Cross-Platform Executables + +We use `pkg` to package the source code into executables. + +This requires a specific version of `pkg` and also `node-gyp-build`. + +Configuration for `pkg` is done in: + +* `package.json` - Pins `pkg` and `node-gyp-build`, and configures assets and scripts. +* `utils.nix` - Pins `pkg` for Nix usage +* `release.nix` - Build expressions for executables + +## Deployment + +Image deployments are done automatically through the CI/CD. However manual scripts are available below for deployment. + +### Deploying to AWS ECR: + +#### Using skopeo + +```sh +tag='manual' +registry_image='015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + +# Authenticates skopeo +aws ecr get-login-password \ + | skopeo login \ + --username AWS \ + --password-stdin \ + "$registry_image" + +build="$(nix-build ./release.nix --attr docker)" +# This will push both the default image tag and the latest tag +./scripts/deploy-image.sh "$build" "$tag" "$registry_image" +``` + +#### Using docker + +```sh +tag='manual' +registry_image='015248367786.dkr.ecr.ap-southeast-2.amazonaws.com/polykey' + +aws ecr get-login-password \ + | docker login \ + --username AWS \ + --password-stdin \ + "$registry_image" + +build="$(nix-build ./release.nix --attr docker)" +loaded="$(docker load --input "$build")" +image_name="$(cut -d':' -f2 <<< "$loaded" | tr -d ' ')" +default_tag="$(cut -d':' -f3 <<< "$loaded")" + +docker tag "${image_name}:${default_tag}" "${registry_image}:${default_tag}" +docker tag "${image_name}:${default_tag}" "${registry_image}:${tag}" +docker tag "${image_name}:${default_tag}" "${registry_image}:latest" + +docker push "${registry_image}:${default_tag}" +docker push "${registry_image}:${tag}" +docker push "${registry_image}:latest" +``` + diff --git a/default.nix b/default.nix new file mode 100644 index 00000000..283de766 --- /dev/null +++ b/default.nix @@ -0,0 +1,57 @@ +{ runCommandNoCC +, callPackage +, jq +}: + +let + utils = callPackage ./utils.nix {}; + drv = runCommandNoCC + "${utils.basename}-${utils.node2nixDev.version}" + { + version = utils.node2nixDev.version; + packageName = utils.node2nixDev.packageName; + } + '' + mkdir -p "$out/lib/node_modules/$packageName" + # copy the package.json + cp \ + "${utils.node2nixDev}/lib/node_modules/$packageName/package.json" \ + "$out/lib/node_modules/$packageName/" + # copy the native addons + if [ -d "${utils.node2nixDev}/lib/node_modules/$packageName/prebuilds" ]; then + cp -r \ + "${utils.node2nixDev}/lib/node_modules/$packageName/prebuilds" \ + "$out/lib/node_modules/$packageName/" + fi + # copy the dist + cp -r \ + "${utils.node2nixDev}/lib/node_modules/$packageName/dist" \ + "$out/lib/node_modules/$packageName/" + # copy over the production dependencies + if [ -d "${utils.node2nixProd}/lib/node_modules" ]; then + cp -r \ + "${utils.node2nixProd}/lib/node_modules" \ + "$out/lib/node_modules/$packageName/" + fi + # symlink bin executables + if [ \ + "$(${jq}/bin/jq 'has("bin")' "$out/lib/node_modules/$packageName/package.json")" \ + == \ + "true" \ + ]; then + mkdir -p "$out/bin" + while IFS= read -r bin_name && IFS= read -r bin_path; do + # make files executable + chmod a+x "$out/lib/node_modules/$packageName/$bin_path" + # create the symlink + ln -s \ + "../lib/node_modules/$packageName/$bin_path" \ + "$out/bin/$bin_name" + done < <( + ${jq}/bin/jq -r 'select(.bin != null) | .bin | to_entries[] | (.key, .value)' \ + "$out/lib/node_modules/$packageName/package.json" + ) + fi + ''; +in + drv diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 00000000..e2ac6616 --- /dev/null +++ b/docs/.nojekyll @@ -0,0 +1 @@ +TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false. \ No newline at end of file diff --git a/docs/assets/highlight.css b/docs/assets/highlight.css new file mode 100644 index 00000000..1c8c07d5 --- /dev/null +++ b/docs/assets/highlight.css @@ -0,0 +1,78 @@ +:root { + --light-hl-0: #795E26; + --dark-hl-0: #DCDCAA; + --light-hl-1: #000000; + --dark-hl-1: #D4D4D4; + --light-hl-2: #0000FF; + --dark-hl-2: #569CD6; + --light-hl-3: #A31515; + --dark-hl-3: #CE9178; + --light-hl-4: #001080; + --dark-hl-4: #9CDCFE; + --light-hl-5: #008000; + --dark-hl-5: #6A9955; + --light-hl-6: #AF00DB; + --dark-hl-6: #C586C0; + --light-hl-7: #0451A5; + --dark-hl-7: #9CDCFE; + --light-code-background: #FFFFFF; + --dark-code-background: #1E1E1E; +} + +@media (prefers-color-scheme: light) { :root { + --hl-0: var(--light-hl-0); + --hl-1: var(--light-hl-1); + --hl-2: var(--light-hl-2); + --hl-3: var(--light-hl-3); + --hl-4: var(--light-hl-4); + --hl-5: var(--light-hl-5); + --hl-6: var(--light-hl-6); + --hl-7: var(--light-hl-7); + --code-background: var(--light-code-background); +} } + +@media (prefers-color-scheme: dark) { :root { + --hl-0: var(--dark-hl-0); + --hl-1: var(--dark-hl-1); + --hl-2: var(--dark-hl-2); + --hl-3: var(--dark-hl-3); + --hl-4: var(--dark-hl-4); + --hl-5: var(--dark-hl-5); + --hl-6: var(--dark-hl-6); + --hl-7: var(--dark-hl-7); + --code-background: var(--dark-code-background); +} } + +:root[data-theme='light'] { + --hl-0: var(--light-hl-0); + --hl-1: var(--light-hl-1); + --hl-2: var(--light-hl-2); + --hl-3: var(--light-hl-3); + --hl-4: var(--light-hl-4); + --hl-5: var(--light-hl-5); + --hl-6: var(--light-hl-6); + --hl-7: var(--light-hl-7); + --code-background: var(--light-code-background); +} + +:root[data-theme='dark'] { + --hl-0: var(--dark-hl-0); + --hl-1: var(--dark-hl-1); + --hl-2: var(--dark-hl-2); + --hl-3: var(--dark-hl-3); + --hl-4: var(--dark-hl-4); + --hl-5: var(--dark-hl-5); + --hl-6: var(--dark-hl-6); + --hl-7: var(--dark-hl-7); + --code-background: var(--dark-code-background); +} + +.hl-0 { color: var(--hl-0); } +.hl-1 { color: var(--hl-1); } +.hl-2 { color: var(--hl-2); } +.hl-3 { color: var(--hl-3); } +.hl-4 { color: var(--hl-4); } +.hl-5 { color: var(--hl-5); } +.hl-6 { color: var(--hl-6); } +.hl-7 { color: var(--hl-7); } +pre, code { background: var(--code-background); } diff --git a/docs/assets/main.js b/docs/assets/main.js new file mode 100644 index 00000000..f7c83669 --- /dev/null +++ b/docs/assets/main.js @@ -0,0 +1,58 @@ +"use strict"; +"use strict";(()=>{var Qe=Object.create;var ae=Object.defineProperty;var Pe=Object.getOwnPropertyDescriptor;var Ce=Object.getOwnPropertyNames;var Oe=Object.getPrototypeOf,Re=Object.prototype.hasOwnProperty;var _e=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var Me=(t,e,n,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of Ce(e))!Re.call(t,i)&&i!==n&&ae(t,i,{get:()=>e[i],enumerable:!(r=Pe(e,i))||r.enumerable});return t};var De=(t,e,n)=>(n=t!=null?Qe(Oe(t)):{},Me(e||!t||!t.__esModule?ae(n,"default",{value:t,enumerable:!0}):n,t));var de=_e((ce,he)=>{(function(){var t=function(e){var n=new t.Builder;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),n.searchPipeline.add(t.stemmer),e.call(n,n),n.build()};t.version="2.3.9";t.utils={},t.utils.warn=function(e){return function(n){e.console&&console.warn&&console.warn(n)}}(this),t.utils.asString=function(e){return e==null?"":e.toString()},t.utils.clone=function(e){if(e==null)return e;for(var n=Object.create(null),r=Object.keys(e),i=0;iGenerated using TypeDoc
Note that JavaScript libraries are not packaged in Nix. Only JavaScript applications are.
+Building the package:
+nix-build -E '(import ./pkgs.nix {}).callPackage ./default.nix {}'
+
+Building the releases:
+nix-build ./release.nix --attr application
nix-build ./release.nix --attr docker
nix-build ./release.nix --attr package.linux.x64.elf
nix-build ./release.nix --attr package.windows.x64.exe
nix-build ./release.nix --attr package.macos.x64.macho
+
+Install into Nix user profile:
+nix-env -f ./release.nix --install --attr application
+
+Install into Docker:
+loaded="$(docker load --input "$(nix-build ./release.nix --attr docker)")"
image="$(cut -d' ' -f3 <<< "$loaded")"
docker run -it "$image"
+
+
+
+ Run nix-shell
, and once you're inside, you can use:
# install (or reinstall packages from package.json)
npm install
# build the dist
npm run build
# run the repl (this allows you to import from ./src)
npm run ts-node
# run the tests
npm run test
# lint the source code
npm run lint
# automatically fix the source
npm run lintfix
+
+
+
+ When calling executables in development, use this style:
+npm run typescript-demo-lib -- p1 p2 p3
+
+The --
is necessary to make npm
understand that the parameters are for your own executable, and not parameters to npm
.
$ npm run ts-node
> import fs from 'fs';
> fs
> import { Library } from '@';
> Library
> import Library as Library2 from './src/lib/Library';
+
+You can also create test files in ./src
, and run them with npm run ts-node ./src/test.ts
.
This allows you to test individual pieces of typescript code, and it makes it easier when doing large scale architecting of TypeScript code.
+ + +Due to https://github.com/microsoft/TypeScript/issues/10866, you cannot use path aliases without a bundler like Webpack to further transform the generated JavaScript code in order to resolve the path aliases. Because this is a simple library demonstration, there's no need to use a bundler. In fact, for such libraries, it is far more efficient to not bundle the code.
+However, we have left the path alias configuration in tsconfig.json
, jest.config.js
and in the tests we are making use of the @
alias.
When developing on multiple NPM packages, it can be easier to use npm link
so that changes are immediately reflected rather than repeatedly publishing packages. To do this, you need to use npm link
. After linking a local directory, you need to provide tsconfig.json
paths so TypeScript compiler can find the right files.
For example when linking @matrixai/db
located in ../js-db
:
npm link ../js-db
+
+You would need to add these paths to tsconfig.json
:
"paths": {
"@": ["index"],
"@/*": ["*"],
"@matrixai/db": ["../node_modules/@matrixai/db/src"],
"@matrixai/db/*": ["../node_modules/@matrixai/db/src/*"]
},
+
+
+
+ There are some nuances when packaging with native modules. +Included native modules are level witch include leveldown and utp-native.
+If a module is not set to public then pkg defaults to including it as bytecode.
+To avoid this breaking with the --no-bytecode
flag we need to add --public-packages "*"
To get leveldown to work with pkg we need to include the prebuilds with the executable.
+after building with pkg you need to copy from node_modules/leveldown/prebuilds
-> path_to_executable/prebuilds
+You only need to include the prebuilds for the arch you are targeting. e.g. for linux-x64 you need prebuild/linux-x64
.
The folder structure for the executable should look like this.
+Including utp-native is simpler, you just need to add it as an asset for pkg. +Add the following lines to the package.json.
+"pkg": {
"assets": "node_modules/utp-native/**/*"
}
+
+
+
+ To make sure that the worker threads work properly you need to include the compiled worker scripts as an asset.
+This can be fixed by adding the following to package.json
"pkg": {
"assets": "dist/bin/worker.js"
}
+
+If you need to include multiple assets then add them as an array.
+"pkg": {
"assets": [
"node_modules/utp-native/**/*",
"dist/bin/worker.js"
]
}
+
+
+
+ npm run docs
+
+See the docs at: https://matrixai.github.io/TypeScript-Demo-Lib/
+ + +Publishing is handled automatically by the staging pipeline.
+Prerelease:
+# npm login
npm version prepatch --preid alpha # premajor/preminor/prepatch
git push --follow-tags
+
+Release:
+# npm login
npm version patch # major/minor/patch
git push --follow-tags
+
+Manually:
+# npm login
npm version patch # major/minor/patch
npm run build
npm publish --access public
git push
git push --tags
+
+Generated using TypeDoc
Generated using TypeDoc