feat!: support DNS over HTTPS and DNS-JSON over HTTPS (#55)

Co-authored-by: Russell Dempsey <[email protected]>

BREAKING CHANGE: alters the options object passed to the `ipns` factory function

Author: achingbrain

.gitignore

@@ -7,3 +7,6 @@ node_modules
 package-lock.json
 yarn.lock
 .vscode
+.env
+.envrc
+.tool-versions

package.json

@@ -36,7 +36,7 @@
     "docs:no-publish": "NODE_OPTIONS=--max_old_space_size=8192 aegir docs --publish false -- --exclude packages/interop"
   },
   "devDependencies": {
-    "aegir": "^41.0.0",
+    "aegir": "^41.1.14",
     "npm-run-all": "^4.1.5"
   },
   "type": "module",

packages/interop/.aegir.js

@@ -12,7 +12,7 @@ export default {
           host: '127.0.0.1',
           port: ipfsdPort
         }, {
-          ipfsBin: (await import('go-ipfs')).default.path(),
+          ipfsBin: (await import('kubo')).default.path(),
           kuboRpcModule: kuboRpcClient,
           ipfsOptions: {
             config: {

packages/interop/package.json

@@ -60,16 +60,15 @@
     "@libp2p/peer-id-factory": "^3.0.3",
     "@libp2p/tcp": "^8.0.4",
     "@libp2p/websockets": "^7.0.4",
-    "aegir": "^41.0.0",
     "blockstore-core": "^4.0.1",
     "datastore-core": "^9.0.3",
-    "go-ipfs": "^0.22.0",
     "helia": "^2.0.1",
     "ipfsd-ctl": "^13.0.0",
     "ipns": "^7.0.1",
     "it-all": "^3.0.2",
     "it-last": "^3.0.1",
     "it-map": "^3.0.3",
+    "kubo": "^0.24.0",
     "kubo-rpc-client": "^3.0.0",
     "libp2p": "^0.46.6",
     "merge-options": "^3.0.4",
@@ -79,7 +78,7 @@
   },
   "browser": {
     "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js",
-    "go-ipfs": false
+    "kubo": false
   },
   "private": true
 }

packages/interop/test/dht.spec.ts

@@ -131,9 +131,11 @@ keyTypes.forEach(type => {
         message: 'Kubo could not find Helia on the DHT'
       })
 
-      name = ipns(helia, [
-        dht(helia)
-      ])
+      name = ipns(helia, {
+        routers: [
+          dht(helia)
+        ]
+      })
     }
 
     afterEach(async () => {

packages/interop/test/fixtures/create-kubo.ts

@@ -1,15 +1,13 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
-// @ts-ignore no types - TODO: remove me once the next version of npm-go-ipfs has shipped
-import * as goIpfs from 'go-ipfs'
 import { type Controller, type ControllerOptions, createController } from 'ipfsd-ctl'
+import * as kubo from 'kubo'
 import * as kuboRpcClient from 'kubo-rpc-client'
 import mergeOptions from 'merge-options'
 import { isElectronMain, isNode } from 'wherearewe'
 
 export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise<Controller> {
   const opts = mergeOptions({
     kuboRpcModule: kuboRpcClient,
-    ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined,
+    ipfsBin: isNode || isElectronMain ? kubo.path() : undefined,
     test: true,
     endpoint: process.env.IPFSD_SERVER,
     ipfsOptions: {

packages/interop/test/pubsub.spec.ts

@@ -52,9 +52,11 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => {
       // connect the two nodes
       await connect(helia, kubo, '/meshsub/1.1.0')
 
-      name = ipns(helia, [
-        pubsub(helia)
-      ])
+      name = ipns(helia, {
+        routers: [
+          pubsub(helia)
+        ]
+      })
     })
 
     afterEach(async () => {

packages/ipns/package.json

@@ -46,6 +46,10 @@
     "./routing": {
       "types": "./dist/src/routing/index.d.ts",
       "import": "./dist/src/routing/index.js"
+    },
+    "./dns-resolvers": {
+      "types": "./dist/src/dns-resolvers/index.d.ts",
+      "import": "./dist/src/dns-resolvers/index.js"
     }
   },
   "eslintConfig": {
@@ -155,10 +159,11 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@libp2p/interface": "^0.1.2",
     "@libp2p/kad-dht": "^10.0.11",
     "@libp2p/logger": "^3.0.2",
     "@libp2p/peer-id": "^3.0.2",
+    "dns-over-http-resolver": "^2.1.3",
+    "dns-packet": "^5.6.0",
     "hashlru": "^2.3.0",
     "interface-datastore": "^8.0.0",
     "ipns": "^7.0.1",
@@ -169,13 +174,17 @@
     "uint8arrays": "^4.0.3"
   },
   "devDependencies": {
+    "@libp2p/interface": "^0.1.4",
     "@libp2p/peer-id-factory": "^3.0.3",
-    "aegir": "^41.0.0",
+    "@types/dns-packet": "^5.6.4",
     "datastore-core": "^9.0.3",
     "sinon": "^17.0.0",
     "sinon-ts": "^1.0.0"
   },
   "browser": {
-    "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js"
+    "./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js"
+  },
+  "typedoc": {
+    "entryPoint": "./src/index.ts"
   }
 }

packages/ipns/src/dns-resolvers/default.ts

@@ -0,0 +1,9 @@
+import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
+import resolve from './resolver.js'
+import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'
+
+export function defaultResolver (): DNSResolver {
+  return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
+  }
+}

packages/ipns/src/dns-resolvers/dns-json-over-https.ts

@@ -0,0 +1,90 @@
+/* eslint-env browser */
+
+import PQueue from 'p-queue'
+import { CustomProgressEvent } from 'progress-events'
+import { type DNSResponse, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink, ipfsPathAndAnswer } from '../utils/dns.js'
+import { TLRU } from '../utils/tlru.js'
+import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'
+
+// Avoid sending multiple queries for the same hostname by caching results
+const cache = new TLRU<string>(1000)
+// This TTL will be used if the remote service does not return one
+const ttl = 60 * 1000
+
+/**
+ * Uses the RFC 8427 'application/dns-json' content-type to resolve DNS queries.
+ *
+ * Supports and server that uses the same schema as Google's DNS over HTTPS
+ * resolver.
+ *
+ * This resolver needs fewer dependencies than the regular DNS-over-HTTPS
+ * resolver so can result in a smaller bundle size and consequently is preferred
+ * for browser use.
+ *
+ * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/
+ * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
+ * @see https://dnsprivacy.org/public_resolvers/
+ * @see https://datatracker.ietf.org/doc/html/rfc8427
+ */
+export function dnsJsonOverHttps (url: string): DNSResolver {
+  // browsers limit concurrent connections per host,
+  // we don't want preload calls to exhaust the limit (~6)
+  const httpQueue = new PQueue({ concurrency: 4 })
+
+  const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    const searchParams = new URLSearchParams()
+    searchParams.set('name', fqdn)
+    searchParams.set('type', 'TXT')
+
+    const query = searchParams.toString()
+
+    // try cache first
+    if (options.nocache !== true && cache.has(query)) {
+      const response = cache.get(query)
+
+      if (response != null) {
+        options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
+        return response
+      }
+    }
+
+    options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))
+
+    // query DNS-JSON over HTTPS server
+    const response = await httpQueue.add(async () => {
+      const res = await fetch(`${url}?${searchParams}`, {
+        headers: {
+          accept: 'application/dns-json'
+        },
+        signal: options.signal
+      })
+
+      if (res.status !== 200) {
+        throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)
+      }
+
+      const query = new URL(res.url).search.slice(1)
+      const json: DNSResponse = await res.json()
+
+      options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))
+
+      const { ipfsPath, answer } = ipfsPathAndAnswer(fqdn, json)
+
+      cache.set(query, ipfsPath, answer.TTL ?? ttl)
+
+      return ipfsPath
+    }, {
+      signal: options.signal
+    })
+
+    if (response == null) {
+      throw new Error('No DNS response received')
+    }
+
+    return response
+  }
+
+  return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
+    return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
+  }
+}