Merge pull request #694 from ipfs-shipyard/feat/actions-on-dnslink-sites

When we introduced option to opt-out from redirect per site (#687), it came with a side effect of removing IPFS context actions. 

This PR (aka Show IPFS Actions on DNSLink Sites):

- Adds context actions on DNSLink sites (when redirect is disabled)
- Adds a bunch of tests
- Tweaks behavior of pin/unpin via browser action menu
- Works around missing dnslink resolver under js

add-on/src/lib/copier.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const { safeIpfsPath, trimHashAndSearch } = require('./ipfs-path')
 const { findValueForContext } = require('./context-menus')
 
 async function copyTextToClipboard (text, notify) {
@@ -32,21 +31,19 @@ async function copyTextToClipboard (text, notify) {
   }
 }
 
-function createCopier (getState, getIpfs, notify) {
+function createCopier (notify, ipfsPathValidator) {
   return {
     async copyCanonicalAddress (context, contextType) {
       const url = await findValueForContext(context, contextType)
-      const rawIpfsAddress = safeIpfsPath(url)
-      await copyTextToClipboard(rawIpfsAddress, notify)
+      const ipfsPath = ipfsPathValidator.resolveToIpfsPath(url)
+      await copyTextToClipboard(ipfsPath, notify)
     },
 
     async copyRawCid (context, contextType) {
+      const url = await findValueForContext(context, contextType)
       try {
-        const ipfs = getIpfs()
-        const url = await findValueForContext(context, contextType)
-        const rawIpfsAddress = trimHashAndSearch(safeIpfsPath(url))
-        const directCid = (await ipfs.resolve(rawIpfsAddress, { recursive: true, dhtt: '5s', dhtrc: 1 })).split('/')[2]
-        await copyTextToClipboard(directCid, notify)
+        const cid = await ipfsPathValidator.resolveToCid(url)
+        await copyTextToClipboard(cid, notify)
       } catch (error) {
         console.error('Unable to resolve/copy direct CID:', error.message)
         if (notify) {
@@ -65,9 +62,8 @@ function createCopier (getState, getIpfs, notify) {
 
     async copyAddressAtPublicGw (context, contextType) {
       const url = await findValueForContext(context, contextType)
-      const state = getState()
-      const urlAtPubGw = url.replace(state.gwURLString, state.pubGwURLString)
-      await copyTextToClipboard(urlAtPubGw, notify)
+      const publicUrl = ipfsPathValidator.resolveToPublicUrl(url)
+      await copyTextToClipboard(publicUrl, notify)
     }
   }
 }

add-on/src/lib/dnslink.js

@@ -7,6 +7,8 @@ const PQueue = require('p-queue')
 const { offlinePeerCount } = require('./state')
 const { pathAtHttpGateway } = require('./ipfs-path')
 
+// TODO: add Preferences toggle to disable redirect of DNSLink  websites (while keeping async dnslink lookup)
+
 module.exports = function createDnslinkResolver (getState) {
   // DNSLink lookup result cache
   const cacheOptions = { max: 1000, maxAge: 1000 * 60 * 60 * 12 }

add-on/src/lib/ipfs-companion.js

@@ -5,7 +5,7 @@ const browser = require('webextension-polyfill')
 const toMultiaddr = require('uri-to-multiaddr')
 const { optionDefaults, storeMissingOptions, migrateOptions } = require('./options')
 const { initState, offlinePeerCount } = require('./state')
-const { createIpfsPathValidator, pathAtHttpGateway } = require('./ipfs-path')
+const { createIpfsPathValidator } = require('./ipfs-path')
 const createDnslinkResolver = require('./dnslink')
 const { createRequestModifier, redirectOptOutHint } = require('./ipfs-request')
 const { initIpfsClient, destroyIpfsClient } = require('./ipfs-client')
@@ -56,9 +56,9 @@ module.exports = async function init () {
       }
     }
 
-    copier = createCopier(getState, getIpfs, notify)
     dnslinkResolver = createDnslinkResolver(getState)
-    ipfsPathValidator = createIpfsPathValidator(getState, dnslinkResolver)
+    ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver)
+    copier = createCopier(notify, ipfsPathValidator)
     contextMenus = createContextMenus(getState, runtime, ipfsPathValidator, {
       onAddFromContext,
       onCopyCanonicalAddress: copier.copyCanonicalAddress,
@@ -174,7 +174,7 @@ module.exports = async function init () {
     // console.log((sender.tab ? 'Message from a content script:' + sender.tab.url : 'Message from the extension'), request)
     if (request.pubGwUrlForIpfsOrIpnsPath) {
       const path = request.pubGwUrlForIpfsOrIpnsPath
-      const result = ipfsPathValidator.validIpfsOrIpnsPath(path) ? pathAtHttpGateway(path, state.pubGwURLString) : null
+      const result = ipfsPathValidator.validIpfsOrIpnsPath(path) ? ipfsPathValidator.resolveToPublicUrl(path, state.pubGwURLString) : null
       return Promise.resolve({ pubGwUrlForIpfsOrIpnsPath: result })
     }
   }
@@ -257,7 +257,7 @@ module.exports = async function init () {
     return new Promise((resolve, reject) => {
       const http = new XMLHttpRequest()
       // Make sure preload request is excluded from global redirect
-      const preloadUrl = pathAtHttpGateway(`${path}#${redirectOptOutHint}`, state.pubGwURLString)
+      const preloadUrl = ipfsPathValidator.resolveToPublicUrl(`${path}#${redirectOptOutHint}`, state.pubGwURLString)
       http.open('HEAD', preloadUrl)
       http.onreadystatechange = function () {
         if (this.readyState === this.DONE) {
@@ -699,6 +699,10 @@ module.exports = async function init () {
       return dnslinkResolver
     },
 
+    get ipfsPathValidator () {
+      return ipfsPathValidator
+    },
+
     get notify () {
       return notify
     },

add-on/src/lib/ipfs-path.js

@@ -2,24 +2,34 @@
 /* eslint-env browser */
 
 const IsIpfs = require('is-ipfs')
+const isFQDN = require('is-fqdn')
 
-function safeIpfsPath (urlOrPath) {
+function normalizedIpfsPath (urlOrPath) {
+  let result = urlOrPath
+  // Convert CID-in-subdomain URL to /ipns/<fqdn>/ path
   if (IsIpfs.subdomain(urlOrPath)) {
-    urlOrPath = subdomainToIpfsPath(urlOrPath)
+    result = subdomainToIpfsPath(urlOrPath)
   }
-  // better safe than sorry: https://github.com/ipfs/ipfs-companion/issues/303
-  return decodeURIComponent(urlOrPath.replace(/^.*(\/ip(f|n)s\/.+)$/, '$1'))
+  // Drop everything before the IPFS path
+  result = result.replace(/^.*(\/ip(f|n)s\/.+)$/, '$1')
+  // Remove Unescape special characters
+  // https://github.com/ipfs/ipfs-companion/issues/303
+  result = decodeURIComponent(result)
+  // Return a valid IPFS path or null otherwise
+  return IsIpfs.path(result) ? result : null
 }
-exports.safeIpfsPath = safeIpfsPath
+exports.normalizedIpfsPath = normalizedIpfsPath
 
 function subdomainToIpfsPath (url) {
   if (typeof url === 'string') {
     url = new URL(url)
   }
   const fqdn = url.hostname.split('.')
+  // TODO: support CID split with commas
   const cid = fqdn[0]
+  // TODO: support .ip(f|n)s. being at deeper levels
   const protocol = fqdn[1]
-  return `/${protocol}/${cid}${url.pathname}`
+  return `/${protocol}/${cid}${url.pathname}${url.search}${url.hash}`
 }
 
 function pathAtHttpGateway (path, gatewayUrl) {
@@ -39,34 +49,37 @@ function trimHashAndSearch (urlString) {
 }
 exports.trimHashAndSearch = trimHashAndSearch
 
-function createIpfsPathValidator (getState, dnsLink) {
+function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) {
   const ipfsPathValidator = {
     // Test if URL is a Public IPFS resource
     // (pass validIpfsOrIpnsUrl(url) and not at the local gateway or API)
     publicIpfsOrIpnsResource (url) {
       // exclude custom gateway and api, otherwise we have infinite loops
       if (!url.startsWith(getState().gwURLString) && !url.startsWith(getState().apiURLString)) {
-        return validIpfsOrIpnsUrl(url, dnsLink)
+        return validIpfsOrIpnsUrl(url, dnslinkResolver)
       }
       return false
     },
 
     // Test if URL is a valid IPFS or IPNS
     // (IPFS needs to be a CID, IPNS can be PeerId or have dnslink entry)
     validIpfsOrIpnsUrl (url) {
-      return validIpfsOrIpnsUrl(url, dnsLink)
+      return validIpfsOrIpnsUrl(url, dnslinkResolver)
     },
 
     // Same as validIpfsOrIpnsUrl (url) but for paths
     // (we have separate methods to avoid 'new URL' where possible)
     validIpfsOrIpnsPath (path) {
-      return validIpfsOrIpnsPath(path, dnsLink)
+      return validIpfsOrIpnsPath(path, dnslinkResolver)
     },
 
     // Test if actions such as 'copy URL', 'pin/unpin' should be enabled for the URL
-    // TODO: include hostname check for DNSLink and display option to copy CID even if no redirect
     isIpfsPageActionsContext (url) {
-      return (IsIpfs.url(url) && !url.startsWith(getState().apiURLString)) || IsIpfs.subdomain(url)
+      return Boolean(url && !url.startsWith(getState().apiURLString) && (
+        IsIpfs.url(url) ||
+        IsIpfs.subdomain(url) ||
+        dnslinkResolver.cachedDnslink(new URL(url).hostname)
+      ))
     },
 
     // Test if actions such as 'per site redirect toggle' should be enabled for the URL
@@ -77,12 +90,146 @@ function createIpfsPathValidator (getState, dnsLink) {
         (url.startsWith('http') && // hide on non-HTTP pages
          !url.startsWith(state.gwURLString) && // hide on /ipfs/*
           !url.startsWith(state.apiURLString))) // hide on api port
+    },
+
+    // Resolve URL or path to HTTP URL:
+    // - IPFS paths are attached to HTTP Gateway root
+    // - URL of DNSLinked websites are returned as-is
+    // The purpose of this resolver is to always return a meaningful, publicly
+    // accessible URL that can be accessed without the need of IPFS client.
+    resolveToPublicUrl (urlOrPath, optionalGatewayUrl) {
+      const input = urlOrPath
+      // CID-in-subdomain is good as-is
+      if (IsIpfs.subdomain(input)) return input
+      // IPFS Paths should be attached to the public gateway
+      const ipfsPath = normalizedIpfsPath(input)
+      const gateway = optionalGatewayUrl || getState().pubGwURLString
+      if (ipfsPath) return pathAtHttpGateway(ipfsPath, gateway)
+      // Return original URL (eg. DNSLink domains) or null if not an URL
+      return input.startsWith('http') ? input : null
+    },
+
+    // Resolve URL or path to IPFS Path:
+    // - The path can be /ipfs/ or /ipns/
+    // - Keeps pathname + ?search + #hash from original URL
+    // - Returns null if no valid path can be produced
+    // The purpose of this resolver is to return a valid IPFS path
+    // that can be accessed with IPFS client.
+    resolveToIpfsPath (urlOrPath) {
+      const input = urlOrPath
+      // Try to normalize to IPFS path (gateway path or CID-in-subdomain)
+      const ipfsPath = normalizedIpfsPath(input)
+      if (ipfsPath) return ipfsPath
+      // Check URL for DNSLink
+      if (!input.startsWith('http')) return null
+      const { hostname } = new URL(input)
+      const dnslink = dnslinkResolver.cachedDnslink(hostname)
+      if (dnslink) {
+        // Return full IPNS path (keeps pathname + ?search + #hash)
+        return dnslinkResolver.convertToIpnsPath(input)
+      }
+      // No IPFS path by this point
+      return null
+    },
+
+    // Resolve URL or path to Immutable IPFS Path:
+    // - Same as resolveToIpfsPath, but the path is always immutable /ipfs/
+    // - Keeps pathname + ?search + #hash from original URL
+    // - Returns null if no valid path can be produced
+    // The purpose of this resolver is to return immutable /ipfs/ address
+    // even if /ipns/ is present in its input.
+    async resolveToImmutableIpfsPath (urlOrPath) {
+      const path = ipfsPathValidator.resolveToIpfsPath(urlOrPath)
+      // Fail fast if no IPFS Path
+      if (!path) return null
+      // Resolve /ipns/ → /ipfs/
+      if (IsIpfs.ipnsPath(path)) {
+        const labels = path.split('/')
+        // We resolve /ipns/<fqdn> as value in DNSLink cache may be out of date
+        const ipnsRoot = `/ipns/${labels[2]}`
+
+        // js-ipfs v0.34 does not support DNSLinks in ipfs.name.resolve: https://github.com/ipfs/js-ipfs/issues/1918
+        // TODO: remove ipfsNameResolveWithDnslinkFallback when js-ipfs implements DNSLink support in ipfs.name.resolve
+        const ipfsNameResolveWithDnslinkFallback = async (resolve) => {
+          try {
+            return await resolve()
+          } catch (err) {
+            const fqdn = ipnsRoot.replace(/^.*\/ipns\/([^/]+).*/, '$1')
+            if (err.message === 'Non-base58 character' && isFQDN(fqdn)) {
+              // js-ipfs without dnslink support, fallback to the value read from DNSLink
+              const dnslink = dnslinkResolver.readAndCacheDnslink(fqdn)
+              if (dnslink) {
+                // swap problematic /ipns/{fqdn} with /ipfs/{cid} and retry lookup
+                const safePath = trimDoubleSlashes(ipnsRoot.replace(/^.*(\/ipns\/[^/]+)/, dnslink))
+                if (ipnsRoot !== safePath) {
+                  return ipfsPathValidator.resolveToImmutableIpfsPath(safePath)
+                }
+              }
+            }
+            throw err
+          }
+        }
+        const result = await ipfsNameResolveWithDnslinkFallback(async () =>
+          // dhtt/dhtrc optimize for lookup time
+          getIpfs().name.resolve(ipnsRoot, { recursive: true, dhtt: '5s', dhtrc: 1 })
+        )
+
+        // Old API returned object, latest one returns string ¯\_(ツ)_/¯
+        const ipfsRoot = result.Path ? result.Path : result
+        // Return original path with swapped root (keeps pathname + ?search + #hash)
+        return path.replace(ipnsRoot, ipfsRoot)
+      }
+      // Return /ipfs/ path
+      return path
+    },
+
+    // Resolve URL or path to a raw CID:
+    // - Result is the direct CID
+    // - Ignores ?search and #hash from original URL
+    // - Returns null if no CID can be produced
+    // The purpose of this resolver is to return direct CID without anything else.
+    async resolveToCid (urlOrPath) {
+      const path = ipfsPathValidator.resolveToIpfsPath(urlOrPath)
+      // Fail fast if no IPFS Path
+      if (!path) return null
+      // Drop unused parts
+      const rawPath = trimHashAndSearch(path)
+
+      // js-ipfs v0.34 does not support DNSLinks in ipfs.resolve: https://github.com/ipfs/js-ipfs/issues/1918
+      // TODO: remove ipfsResolveWithDnslinkFallback when js-ipfs implements DNSLink support in ipfs.resolve
+      const ipfsResolveWithDnslinkFallback = async (resolve) => {
+        try {
+          return await resolve()
+        } catch (err) {
+          const fqdn = rawPath.replace(/^.*\/ipns\/([^/]+).*/, '$1')
+          if (err.message === 'resolve non-IPFS names is not implemented' && isFQDN(fqdn)) {
+            // js-ipfs without dnslink support, fallback to the value read from DNSLink
+            const dnslink = dnslinkResolver.readAndCacheDnslink(fqdn)
+            if (dnslink) {
+              // swap problematic /ipns/{fqdn} with /ipfs/{cid} and retry lookup
+              const safePath = trimDoubleSlashes(rawPath.replace(/^.*(\/ipns\/[^/]+)/, dnslink))
+              if (rawPath !== safePath) {
+                const result = await ipfsPathValidator.resolveToCid(safePath)
+                // return in format of ipfs.resolve()
+                return IsIpfs.cid(result) ? `/ipfs/${result}` : result
+              }
+            }
+          }
+          throw err
+        }
+      }
+      const result = await ipfsResolveWithDnslinkFallback(async () =>
+        // dhtt/dhtrc optimize for lookup time
+        getIpfs().resolve(rawPath, { recursive: true, dhtt: '5s', dhtrc: 1 })
+      )
+
+      const directCid = IsIpfs.ipfsPath(result) ? result.split('/')[2] : result
+      return directCid
     }
   }
 
   return ipfsPathValidator
 }
-
 exports.createIpfsPathValidator = createIpfsPathValidator
 
 function validIpfsOrIpnsUrl (url, dnsLink) {
@@ -122,6 +269,7 @@ function validIpnsPath (path, dnsLink) {
       return true
     }
     // then see if there is an DNSLink entry for 'ipnsRoot' hostname
+    // TODO: use dnslink cache only
     if (dnsLink.readAndCacheDnslink(ipnsRoot)) {
       // console.log('==> IPNS for FQDN with valid dnslink: ', ipnsRoot)
       return true

add-on/src/lib/ipfs-request.js

@@ -3,7 +3,7 @@
 
 const LRU = require('lru-cache')
 const IsIpfs = require('is-ipfs')
-const { safeIpfsPath, pathAtHttpGateway } = require('./ipfs-path')
+const { pathAtHttpGateway } = require('./ipfs-path')
 const redirectOptOutHint = 'x-ipfs-companion-no-redirect'
 const recoverableErrors = new Set([
   // Firefox
@@ -127,7 +127,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
         }
         // Detect valid /ipfs/ and /ipns/ on any site
         if (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) && isSafeToRedirect(request, runtime)) {
-          return redirectToGateway(request.url, state, dnslinkResolver)
+          return redirectToGateway(request.url, state, ipfsPathValidator)
         }
         // Detect dnslink using heuristics enabled in Preferences
         if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
@@ -321,7 +321,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
               return dnslinkRedirect
             }
           }
-          return redirectToGateway(request.url, state, dnslinkResolver)
+          return redirectToGateway(request.url, state, ipfsPathValidator)
         }
 
         // Detect X-Ipfs-Path Header and upgrade transport to IPFS:
@@ -368,7 +368,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
                 // redirect only if anything changed
                 if (newUrl !== request.url) {
                   console.log(`[ipfs-companion] onHeadersReceived: normalized ${request.url} to  ${newUrl}`)
-                  return redirectToGateway(newUrl, state, dnslinkResolver)
+                  return redirectToGateway(newUrl, state, ipfsPathValidator)
                 }
               }
             }
@@ -426,11 +426,11 @@ exports.redirectOptOutHint = redirectOptOutHint
 exports.createRequestModifier = createRequestModifier
 exports.onHeadersReceivedRedirect = onHeadersReceivedRedirect
 
-function redirectToGateway (requestUrl, state, dnslinkResolver) {
+function redirectToGateway (requestUrl, state, ipfsPathValidator) {
   // TODO: redirect to `ipfs://` if hasNativeProtocolHandler === true
   const gateway = state.ipfsNodeType === 'embedded' ? state.pubGwURLString : state.gwURLString
-  const path = safeIpfsPath(requestUrl)
-  return { redirectUrl: pathAtHttpGateway(path, gateway) }
+  const redirectUrl = ipfsPathValidator.resolveToPublicUrl(requestUrl, gateway)
+  return { redirectUrl }
 }
 
 function isSafeToRedirect (request, runtime) {