diff --git a/.github/dictionary.txt b/.github/dictionary.txt index 610adbb25..698481e5f 100644 --- a/.github/dictionary.txt +++ b/.github/dictionary.txt @@ -1 +1,2 @@ dont +fleek diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 7e7c79241..20029dab3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -4,6 +4,7 @@ "packages/car": "4.2.0", "packages/dag-cbor": "4.1.0", "packages/dag-json": "4.1.0", + "packages/dnslink": "0.0.0", "packages/helia": "5.5.1", "packages/interface": "5.4.0", "packages/interop": "8.3.0", diff --git a/.release-please.json b/.release-please.json index 6598baff1..767ad3f18 100644 --- a/.release-please.json +++ b/.release-please.json @@ -14,6 +14,7 @@ "packages/car": {}, "packages/dag-cbor": {}, "packages/dag-json": {}, + "packages/dnslink": {}, "packages/helia": {}, "packages/http": {}, "packages/interface": {}, diff --git a/packages/dnslink/CODE_OF_CONDUCT.md b/packages/dnslink/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..6b0fa54c5 --- /dev/null +++ b/packages/dnslink/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Contributor Code of Conduct + +This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) diff --git a/packages/dnslink/LICENSE-APACHE b/packages/dnslink/LICENSE-APACHE new file mode 100644 index 000000000..b09cd7856 --- /dev/null +++ b/packages/dnslink/LICENSE-APACHE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/dnslink/LICENSE-MIT b/packages/dnslink/LICENSE-MIT new file mode 100644 index 000000000..72dc60d84 --- /dev/null +++ b/packages/dnslink/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/dnslink/README.md b/packages/dnslink/README.md new file mode 100644 index 000000000..df4f47144 --- /dev/null +++ b/packages/dnslink/README.md @@ -0,0 +1,171 @@ +

+ + Helia logo + +

+ +# @helia/dnslink + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> DNSLink operations using Helia + +# About + + + +[DNSLink](https://dnslink.dev/) operations using a Helia node. + +## Example - Using custom DNS over HTTPS resolvers + +To use custom resolvers, configure Helia's `dns` option: + +```TypeScript +import { createHelia } from 'helia' +import { dnsLink } from '@helia/dnslink' +import { dns } from '@multiformats/dns' +import { dnsOverHttps } from '@multiformats/dns/resolvers' +import type { DefaultLibp2pServices } from 'helia' +import type { Libp2p } from '@libp2p/interface' + +const node = await createHelia>({ + dns: dns({ + resolvers: { + '.': dnsOverHttps('https://private-dns-server.me/dns-query') + } + }) +}) +const name = dnsLink(node) + +const result = name.resolve('some-domain-with-dnslink-entry.com') +``` + +## Example - Resolving a domain with a dnslink entry + +Calling `resolve` with the `@helia/dnslink` instance: + +```TypeScript +// resolve a CID from a TXT record in a DNS zone file, using the default +// resolver for the current platform eg: +// > dig _dnslink.ipfs.tech TXT +// ;; ANSWER SECTION: +// _dnslink.ipfs.tech. 60 IN CNAME _dnslink.ipfs-tech.on.fleek.co. +// _dnslink.ipfs-tech.on.fleek.co. 120 IN TXT "dnslink=/ipfs/bafybe..." + +import { createHelia } from 'helia' +import { dnsLink } from '@helia/dnslink' + +const node = await createHelia() +const name = dnsLink(node) + +const { answer } = await name.resolve('blog.ipfs.tech') + +console.info(answer) +// { data: '/ipfs/bafybe...' } +``` + +## Example - Using DNS-Over-HTTPS + +This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This +uses binary DNS records so requires extra dependencies to process the +response which can increase browser bundle sizes. + +If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. + +```TypeScript +import { createHelia } from 'helia' +import { dnsLink } from '@helia/dnslink' +import { dns } from '@multiformats/dns' +import { dnsOverHttps } from '@multiformats/dns/resolvers' +import type { DefaultLibp2pServices } from 'helia' +import type { Libp2p } from '@libp2p/interface' + +const node = await createHelia>({ + dns: dns({ + resolvers: { + '.': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + } + }) +}) +const name = dnsLink(node) + +const result = await name.resolve('blog.ipfs.tech') +``` + +## Example - Using DNS-JSON-Over-HTTPS + +DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can +result in a smaller browser bundle due to the response being plain JSON. + +```TypeScript +import { createHelia } from 'helia' +import { dnsLink } from '@helia/dnslink' +import { dns } from '@multiformats/dns' +import { dnsJsonOverHttps } from '@multiformats/dns/resolvers' +import type { DefaultLibp2pServices } from 'helia' +import type { Libp2p } from '@libp2p/interface' + +const node = await createHelia>({ + dns: dns({ + resolvers: { + '.': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + } + }) +}) +const name = dnsLink(node) + +const result = await name.resolve('blog.ipfs.tech') +``` + +# Install + +```console +$ npm i @helia/dnslink +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/helia/blob/main/packages/ipns/LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](https://github.com/ipfs/helia/blob/main/packages/ipns/LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/dnslink/package.json b/packages/dnslink/package.json new file mode 100644 index 000000000..3dd997634 --- /dev/null +++ b/packages/dnslink/package.json @@ -0,0 +1,83 @@ +{ + "name": "@helia/dnslink", + "version": "0.0.0", + "description": "DNSLink operations using Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/dnslink#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "doc-check": "aegir doc-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "dependencies": { + "@libp2p/interface": "^3.0.2", + "@libp2p/peer-id": "^6.0.3", + "@multiformats/dns": "^1.0.9", + "multiformats": "^13.4.1", + "progress-events": "^1.0.1" + }, + "devDependencies": { + "@libp2p/logger": "^6.0.5", + "@types/dns-packet": "^5.6.5", + "aegir": "^47.0.22", + "sinon-ts": "^2.0.0" + }, + "browser": { + "./dist/src/dns-resolvers/resolver.js": "./dist/src/dns-resolvers/resolver.browser.js" + }, + "sideEffects": false +} diff --git a/packages/dnslink/src/constants.ts b/packages/dnslink/src/constants.ts new file mode 100644 index 000000000..915cabd1b --- /dev/null +++ b/packages/dnslink/src/constants.ts @@ -0,0 +1 @@ +export const MAX_RECURSIVE_DEPTH = 32 diff --git a/packages/dnslink/src/dnslink.ts b/packages/dnslink/src/dnslink.ts new file mode 100644 index 000000000..68591149f --- /dev/null +++ b/packages/dnslink/src/dnslink.ts @@ -0,0 +1,146 @@ +import { MAX_RECURSIVE_DEPTH, RecordType } from '@multiformats/dns' +import { DNSLinkNotFoundError } from './errors.js' +import { ipfs } from './namespaces/ipfs.ts' +import { ipns } from './namespaces/ipns.ts' +import type { DNSLink as DNSLinkInterface, ResolveDNSLinkOptions, DNSLinkOptions, DNSLinkComponents, DNSLinkResult, DNSLinkNamespace } from './index.js' +import type { Logger } from '@libp2p/interface' +import type { DNS } from '@multiformats/dns' + +export class DNSLink implements DNSLinkInterface { + private readonly dns: DNS + private readonly log: Logger + private readonly namespaces: Record + + constructor (components: DNSLinkComponents, init: DNSLinkOptions = {}) { + this.dns = components.dns + this.log = components.logger.forComponent('helia:dnslink') + this.namespaces = { + ipfs, + ipns, + ...init.namespaces + } + } + + async resolve (domain: string, options: ResolveDNSLinkOptions = {}): Promise { + return this.recursiveResolveDomain(domain, options.maxRecursiveDepth ?? MAX_RECURSIVE_DEPTH, options) + } + + async recursiveResolveDomain (domain: string, depth: number, options: ResolveDNSLinkOptions = {}): Promise { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + // the DNSLink spec says records MUST be stored on the `_dnslink.` subdomain + // so start looking for records there, we will fall back to the bare domain + // if none are found + if (!domain.startsWith('_dnslink.')) { + domain = `_dnslink.${domain}` + } + + try { + return await this.recursiveResolveDnslink(domain, depth, options) + } catch (err: any) { + // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error + if (err.code !== 'ENOTFOUND' && err.code !== 'ENODATA' && err.name !== 'DNSLinkNotFoundError' && err.name !== 'NotFoundError') { + throw err + } + + if (domain.startsWith('_dnslink.')) { + // The supplied domain contains a _dnslink component + // Check the non-_dnslink domain + domain = domain.replace('_dnslink.', '') + } else { + // Check the _dnslink subdomain + domain = `_dnslink.${domain}` + } + + // If this throws then we propagate the error + return this.recursiveResolveDnslink(domain, depth, options) + } + } + + async recursiveResolveDnslink (domain: string, depth: number, options: ResolveDNSLinkOptions = {}): Promise { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + this.log('query %s for TXT and CNAME records', domain) + const txtRecordsResponse = await this.dns.query(domain, { + ...options, + types: [ + RecordType.TXT + ] + }) + + // sort the TXT records to ensure deterministic processing + const txtRecords = (txtRecordsResponse?.Answer ?? []) + .sort((a, b) => a.data.localeCompare(b.data)) + + this.log('found %d TXT records for %s', txtRecords.length, domain) + + for (const answer of txtRecords) { + try { + let result = answer.data + + // strip leading and trailing " characters + if (result.startsWith('"') && result.endsWith('"')) { + result = result.substring(1, result.length - 1) + } + + if (!result.startsWith('dnslink=')) { + // invalid record? + continue + } + + this.log('%s TXT %s', answer.name, result) + + result = result.replace('dnslink=', '') + + // result is now a `/ipfs/` or `/ipns/` string + const [, protocol, domainOrCID] = result.split('/') // e.g. ["", "ipfs", ""] + + if (protocol === 'dnslink') { + // if the result was another DNSLink domain, try to follow it + return await this.recursiveResolveDomain(domainOrCID, depth - 1, options) + } + + const parser = this.namespaces[protocol] + + if (parser == null) { + this.log('unknown protocol "%s" in DNSLink record for domain: %s', protocol, domain) + continue + } + + return parser.parse(result, answer) + } catch (err: any) { + this.log.error('could not parse DNS link record for domain %s, %s', domain, answer.data, err) + } + } + + // no dnslink records found, try CNAMEs + this.log('no DNSLink records found for %s, falling back to CNAME', domain) + + const cnameRecordsResponse = await this.dns.query(domain, { + ...options, + types: [ + RecordType.CNAME + ] + }) + + // sort the CNAME records to ensure deterministic processing + const cnameRecords = (cnameRecordsResponse?.Answer ?? []) + .sort((a, b) => a.data.localeCompare(b.data)) + + this.log('found %d CNAME records for %s', cnameRecords.length, domain) + + for (const cname of cnameRecords) { + try { + return await this.recursiveResolveDomain(cname.data, depth - 1, options) + } catch (err: any) { + this.log.error('domain %s cname %s had no DNSLink records - %e', domain, cname.data, err) + } + } + + throw new DNSLinkNotFoundError(`No DNSLink records found for domain: ${domain}`) + } +} diff --git a/packages/dnslink/src/errors.ts b/packages/dnslink/src/errors.ts new file mode 100644 index 000000000..36d8d725d --- /dev/null +++ b/packages/dnslink/src/errors.ts @@ -0,0 +1,17 @@ +export class DNSLinkNotFoundError extends Error { + static name = 'DNSLinkNotFoundError' + + constructor (message = 'DNSLink not found') { + super(message) + this.name = 'DNSLinkNotFoundError' + } +} + +export class InvalidNamespaceError extends Error { + static name = 'InvalidNamespaceError' + + constructor (message = 'Invalid namespace') { + super(message) + this.name = 'InvalidNamespaceError' + } +} diff --git a/packages/dnslink/src/index.ts b/packages/dnslink/src/index.ts new file mode 100644 index 000000000..c2b571465 --- /dev/null +++ b/packages/dnslink/src/index.ts @@ -0,0 +1,241 @@ +/** + * @packageDocumentation + * + * [DNSLink](https://dnslink.dev/) operations using a Helia node. + * + * @example Using custom DNS over HTTPS resolvers + * + * To use custom resolvers, configure Helia's `dns` option: + * + * ```TypeScript + * import { createHelia } from 'helia' + * import { dnsLink } from '@helia/dnslink' + * import { dns } from '@multiformats/dns' + * import { dnsOverHttps } from '@multiformats/dns/resolvers' + * import type { DefaultLibp2pServices } from 'helia' + * import type { Libp2p } from '@libp2p/interface' + * + * const node = await createHelia>({ + * dns: dns({ + * resolvers: { + * '.': dnsOverHttps('https://private-dns-server.me/dns-query') + * } + * }) + * }) + * const name = dnsLink(node) + * + * const result = name.resolve('some-domain-with-dnslink-entry.com') + * ``` + * + * @example Resolving a domain with a dnslink entry + * + * Calling `resolve` with the `@helia/dnslink` instance: + * + * ```TypeScript + * // resolve a CID from a TXT record in a DNS zone file, using the default + * // resolver for the current platform eg: + * // > dig _dnslink.ipfs.tech TXT + * // ;; ANSWER SECTION: + * // _dnslink.ipfs.tech. 60 IN CNAME _dnslink.ipfs-tech.on.fleek.co. + * // _dnslink.ipfs-tech.on.fleek.co. 120 IN TXT "dnslink=/ipfs/bafybe..." + * + * import { createHelia } from 'helia' + * import { dnsLink } from '@helia/dnslink' + * + * const node = await createHelia() + * const name = dnsLink(node) + * + * const { answer } = await name.resolve('blog.ipfs.tech') + * + * console.info(answer) + * // { data: '/ipfs/bafybe...' } + * ``` + * + * @example Using DNS-Over-HTTPS + * + * This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This + * uses binary DNS records so requires extra dependencies to process the + * response which can increase browser bundle sizes. + * + * If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. + * + * ```TypeScript + * import { createHelia } from 'helia' + * import { dnsLink } from '@helia/dnslink' + * import { dns } from '@multiformats/dns' + * import { dnsOverHttps } from '@multiformats/dns/resolvers' + * import type { DefaultLibp2pServices } from 'helia' + * import type { Libp2p } from '@libp2p/interface' + * + * const node = await createHelia>({ + * dns: dns({ + * resolvers: { + * '.': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * } + * }) + * }) + * const name = dnsLink(node) + * + * const result = await name.resolve('blog.ipfs.tech') + * ``` + * + * @example Using DNS-JSON-Over-HTTPS + * + * DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can + * result in a smaller browser bundle due to the response being plain JSON. + * + * ```TypeScript + * import { createHelia } from 'helia' + * import { dnsLink } from '@helia/dnslink' + * import { dns } from '@multiformats/dns' + * import { dnsJsonOverHttps } from '@multiformats/dns/resolvers' + * import type { DefaultLibp2pServices } from 'helia' + * import type { Libp2p } from '@libp2p/interface' + * + * const node = await createHelia>({ + * dns: dns({ + * resolvers: { + * '.': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') + * } + * }) + * }) + * const name = dnsLink(node) + * + * const result = await name.resolve('blog.ipfs.tech') + * ``` + */ + +import { DNSLink as DNSLinkClass } from './dnslink.js' +import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface' +import type { Answer, DNS, ResolveDnsProgressEvents } from '@multiformats/dns' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +export interface ResolveDNSLinkOptions extends AbortOptions, ProgressOptions { + /** + * Do not query the network for the IPNS record + * + * @default false + */ + offline?: boolean + + /** + * Do not use cached DNS entries + * + * @default false + */ + nocache?: boolean + + /** + * When resolving DNSLink records that resolve to other DNSLink records, limit + * how many times we will recursively resolve them. + * + * @default 32 + */ + maxRecursiveDepth?: number +} + +export interface DNSLinkIPFSResult { + /** + * The resolved record + */ + answer: Answer + + /** + * The IPFS namespace + */ + namespace: 'ipfs' + + /** + * The resolved value + */ + cid: CID + + /** + * If the resolved value is an IPFS path, it will be present here + */ + path: string +} + +export interface DNSLinkIPNSResult { + /** + * The resolved record + */ + answer: Answer + + /** + * The IPFS namespace + */ + namespace: 'ipns' + + /** + * The resolved value + */ + peerId: PeerId + + /** + * If the resolved value is an IPFS path, it will be present here + */ + path: string +} + +export interface DNSLinkOtherResult { + /** + * The resolved record + */ + answer: Answer + + /** + * The IPFS namespace + */ + namespace: string +} + +export type DNSLinkResult = DNSLinkIPFSResult | DNSLinkIPNSResult | DNSLinkOtherResult + +export interface DNSLinkNamespace { + /** + * Return a result parsed from a DNSLink value + */ + parse(value: string, answer: Answer): DNSLinkResult +} + +export interface DNSLink { + /** + * Resolve a CID from a dns-link style IPNS record + * + * @example + * + * ```TypeScript + * import { createHelia } from 'helia' + * import { dnsLink } from '@helia/dnslink' + * + * const helia = await createHelia() + * const name = dnsLink(helia) + * + * const result = await name.resolve('ipfs.io', { + * signal: AbortSignal.timeout(5_000) + * }) + * + * console.info(result) // { answer: ..., value: ... } + * ``` + */ + resolve(domain: string, options?: ResolveDNSLinkOptions): Promise +} + +export interface DNSLinkComponents { + dns: DNS + logger: ComponentLogger +} + +export interface DNSLinkOptions { + /** + * By default `/ipfs/...`, `/ipns/...` and `/dnslink/...` record values are + * supported - to support other prefixes pass other value parsers here + */ + namespaces?: Record +} + +export function dnsLink (components: DNSLinkComponents, options: DNSLinkOptions = {}): DNSLink { + return new DNSLinkClass(components, options) +} diff --git a/packages/dnslink/src/namespaces/ipfs.ts b/packages/dnslink/src/namespaces/ipfs.ts new file mode 100644 index 000000000..de8fcc5e4 --- /dev/null +++ b/packages/dnslink/src/namespaces/ipfs.ts @@ -0,0 +1,22 @@ +import { CID } from 'multiformats/cid' +import { InvalidNamespaceError } from '../errors.ts' +import type { DNSLinkResult, DNSLinkNamespace } from '../index.js' +import type { Answer } from '@multiformats/dns' + +export const ipfs: DNSLinkNamespace = { + parse: (value: string, answer: Answer): DNSLinkResult => { + const [, protocol, cid, ...rest] = value.split('/') + + if (protocol !== 'ipfs') { + throw new InvalidNamespaceError(`Namespace ${protocol} was not "ipfs"`) + } + + // if the result is a CID, we've reached the end of the recursion + return { + namespace: 'ipfs', + cid: CID.parse(cid), + path: rest.length > 0 ? `/${rest.join('/')}` : '', + answer + } + } +} diff --git a/packages/dnslink/src/namespaces/ipns.ts b/packages/dnslink/src/namespaces/ipns.ts new file mode 100644 index 000000000..4a66bce53 --- /dev/null +++ b/packages/dnslink/src/namespaces/ipns.ts @@ -0,0 +1,22 @@ +import { peerIdFromString } from '@libp2p/peer-id' +import { InvalidNamespaceError } from '../errors.ts' +import type { DNSLinkResult, DNSLinkNamespace } from '../index.js' +import type { Answer } from '@multiformats/dns' + +export const ipns: DNSLinkNamespace = { + parse: (value: string, answer: Answer): DNSLinkResult => { + const [, protocol, peerId, ...rest] = value.split('/') + + if (protocol !== 'ipns') { + throw new InvalidNamespaceError(`Namespace ${protocol} was not "ipns"`) + } + + // if the result is a CID, we've reached the end of the recursion + return { + namespace: 'ipns', + peerId: peerIdFromString(peerId), + path: rest.length > 0 ? `/${rest.join('/')}` : '', + answer + } + } +} diff --git a/packages/ipns/test/resolve-dnslink.spec.ts b/packages/dnslink/test/index.spec.ts similarity index 63% rename from packages/ipns/test/resolve-dnslink.spec.ts rename to packages/dnslink/test/index.spec.ts index 8f108b5db..a749067ff 100644 --- a/packages/ipns/test/resolve-dnslink.spec.ts +++ b/packages/dnslink/test/index.spec.ts @@ -1,20 +1,16 @@ /* eslint-env mocha */ -import { generateKeyPair } from '@libp2p/crypto/keys' import { NotFoundError } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' -import { peerIdFromPrivateKey } from '@libp2p/peer-id' +import { peerIdFromString } from '@libp2p/peer-id' import { RecordType } from '@multiformats/dns' import { expect } from 'aegir/chai' -import { MemoryDatastore } from 'datastore-core' import { base36 } from 'multiformats/bases/base36' import { CID } from 'multiformats/cid' import { stubInterface } from 'sinon-ts' -import { ipns } from '../src/index.js' -import type { IPNS } from '../src/index.js' -import type { Routing } from '@helia/interface' -import type { DNS, Answer, DNSResponse } from '@multiformats/dns' -import type { Datastore } from 'interface-datastore' +import { dnsLink } from '../src/index.js' +import type { DNSLink } from '../src/index.js' +import type { Answer, DNS, DNSResponse } from '@multiformats/dns' import type { StubbedInstance } from 'sinon-ts' function dnsResponse (answers: Answer[]): DNSResponse { @@ -30,20 +26,13 @@ function dnsResponse (answers: Answer[]): DNSResponse { } } -describe('resolveDNSLink', () => { - let datastore: Datastore - let heliaRouting: StubbedInstance +describe('dnslink', () => { let dns: StubbedInstance - let name: IPNS + let name: DNSLink beforeEach(async () => { - datastore = new MemoryDatastore() - heliaRouting = stubInterface() - dns = stubInterface() - - name = ipns({ - datastore, - routing: heliaRouting, + dns = stubInterface() + name = dnsLink({ dns, logger: defaultLogger() }) @@ -57,8 +46,9 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true, offline: true }) - expect(result.cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const result = await name.resolve('foobar.baz', { nocache: true, offline: true }) + expect(result).to.have.deep.property('cid', CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')) + expect(result).to.have.property('path', '') }) it('should retry without `_dnslink.` on a domain', async () => { @@ -70,8 +60,9 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true, offline: true }) - expect(result.cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const result = await name.resolve('foobar.baz', { nocache: true, offline: true }) + expect(result).to.have.deep.property('cid', CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')) + expect(result).to.have.property('path', '') }) it('should handle bad records', async () => { @@ -107,8 +98,8 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true, offline: true }) - expect(result.cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const result = await name.resolve('foobar.baz', { nocache: true, offline: true }) + expect(result).to.have.deep.property('cid', CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')) }) it('should handle records wrapped in quotation marks', async () => { @@ -119,8 +110,8 @@ describe('resolveDNSLink', () => { data: '"dnslink=/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true, offline: true }) - expect(result.cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const result = await name.resolve('foobar.baz', { nocache: true, offline: true }) + expect(result).to.have.deep.property('cid', CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')) }) it('should support trailing slash in returned dnslink value', async () => { @@ -133,9 +124,9 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe/' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true }) + const result = await name.resolve('foobar.baz', { nocache: true }) // spellchecker:disable-next-line - expect(result.cid.toString()).to.equal('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe', 'doesn\'t support trailing slashes') + expect(result).to.have.deep.property('cid', CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe'), 'doesn\'t support trailing slashes') }) it('should support paths in returned dnslink value', async () => { @@ -148,39 +139,34 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe/foobar/path/123' }])) - const result = await name.resolveDNSLink('foobar.baz', { nocache: true }) + const result = await name.resolve('foobar.baz', { nocache: true }) // spellchecker:disable-next-line - expect(result.cid.toString()).to.equal('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe', 'doesn\'t support trailing slashes') - expect(result.path).to.equal('foobar/path/123') + expect(result).to.have.deep.property('cid', CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe'), 'doesn\'t support trailing paths') + expect(result).to.have.property('path', '/foobar/path/123') }) it('should resolve recursive dnslink -> /', async () => { - const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const key = await generateKeyPair('Ed25519') - const peerId = peerIdFromPrivateKey(key) + const peerId = peerIdFromString('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([{ name: 'foobar.baz.', TTL: 60, type: RecordType.TXT, - data: `dnslink=/ipns/${peerId}/foobar/path/123` + data: `dnslink=/ipns/${peerId.toString()}/foobar/path/123` }])) - await name.publish(key, cid) - - const result = await name.resolveDNSLink('foobar.baz') + const result = await name.resolve('foobar.baz') if (result == null) { throw new Error('Did not resolve entry') } - expect(result.cid.toString()).to.equal(cid.toV1().toString()) - expect(result.path).to.equal('foobar/path/123') + expect(result).to.have.deep.property('peerId', peerId) + expect(result).to.have.property('path', '/foobar/path/123') }) it('should resolve recursive dnslink -> /', async () => { - const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const key = await generateKeyPair('Ed25519') - const peerId = peerIdFromPrivateKey(key) + const peerId = peerIdFromString('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') const peerIdBase36CID = peerId.toCID().toString(base36) dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([{ name: 'foobar.baz.', @@ -189,21 +175,17 @@ describe('resolveDNSLink', () => { data: `dnslink=/ipns/${peerIdBase36CID}/foobar/path/123` }])) - await name.publish(key, cid) - - const result = await name.resolveDNSLink('foobar.baz') + const result = await name.resolve('foobar.baz') if (result == null) { throw new Error('Did not resolve entry') } - expect(result.cid.toString()).to.equal(cid.toV1().toString()) - expect(result.path).to.equal('foobar/path/123') + expect(result).to.have.deep.property('peerId', peerId) + expect(result).to.have.property('path', '/foobar/path/123') }) it('should follow CNAMES to delegated DNSLink domains', async () => { - const cid = CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe') - const key = await generateKeyPair('Ed25519') dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([{ name: '_dnslink.foobar.baz.', TTL: 60, @@ -217,21 +199,17 @@ describe('resolveDNSLink', () => { // spellchecker:disable-next-line data: 'dnslink=/ipfs/bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe' }])) - - await name.publish(key, cid) - - const result = await name.resolveDNSLink('foobar.baz') + const result = await name.resolve('foobar.baz') if (result == null) { throw new Error('Did not resolve entry') } - expect(result.cid.toString()).to.equal(cid.toV1().toString()) + expect(result).to.have.deep.property('cid', CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe')) }) it('should resolve dnslink namespace', async () => { const cid = CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe') - const key = await generateKeyPair('Ed25519') dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([{ name: '_dnslink.foobar.baz.', TTL: 60, @@ -246,20 +224,16 @@ describe('resolveDNSLink', () => { data: 'dnslink=/ipfs/bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe' }])) - await name.publish(key, cid) - - const result = await name.resolveDNSLink('foobar.baz') + const result = await name.resolve('foobar.baz') if (result == null) { throw new Error('Did not resolve entry') } - expect(result.cid.toString()).to.equal(cid.toV1().toString()) + expect(result).to.have.deep.property('cid', cid) }) it('should include DNS Answer in result', async () => { - const cid = CID.parse('bafybeifcaqowoyito3qvsmbwbiugsu4umlxn4ehu223hvtubbfvwyuxjoe') - const key = await generateKeyPair('Ed25519') const answer = { name: '_dnslink.foobar.baz.', TTL: 60, @@ -269,9 +243,7 @@ describe('resolveDNSLink', () => { } dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([answer])) - await name.publish(key, cid) - - const result = await name.resolveDNSLink('foobar.baz') + const result = await name.resolve('foobar.baz') if (result == null) { throw new Error('Did not resolve entry') @@ -279,4 +251,37 @@ describe('resolveDNSLink', () => { expect(result).to.have.deep.property('answer', answer) }) + + it('should support custom parsers', async () => { + const answer = { + name: '_dnslink.foobar.baz.', + TTL: 60, + type: RecordType.TXT, + // spellchecker:disable-next-line + data: 'dnslink=/hello/world' + } + dns.query.withArgs('_dnslink.foobar.baz').resolves(dnsResponse([answer])) + + name = dnsLink({ + dns, + logger: defaultLogger() + }, { + namespaces: { + hello: { + parse: (value, answer) => { + return { + namespace: 'hello', + value: value.split('/hello/').pop(), + answer + } + } + } + } + }) + + const result = await name.resolve('foobar.baz') + + expect(result).to.have.property('namespace', 'hello') + expect(result).to.have.property('value', 'world') + }) }) diff --git a/packages/dnslink/tsconfig.json b/packages/dnslink/tsconfig.json new file mode 100644 index 000000000..4c0bdf772 --- /dev/null +++ b/packages/dnslink/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface" + } + ] +} diff --git a/packages/dnslink/typedoc.json b/packages/dnslink/typedoc.json new file mode 100644 index 000000000..5ce3e5777 --- /dev/null +++ b/packages/dnslink/typedoc.json @@ -0,0 +1,8 @@ +{ + "readme": "none", + "entryPoints": [ + "./src/index.ts", + "./src/dns-resolvers/index.ts", + "./src/routing/index.ts" + ] +} diff --git a/packages/interface/src/index.ts b/packages/interface/src/index.ts index 1b1dfc566..08b0d4c7c 100644 --- a/packages/interface/src/index.ts +++ b/packages/interface/src/index.ts @@ -17,7 +17,7 @@ import type { Blocks } from './blocks.js' import type { Pins } from './pins.js' import type { Routing } from './routing.js' -import type { AbortOptions, ComponentLogger, Libp2p, Metrics } from '@libp2p/interface' +import type { AbortOptions, ComponentLogger, Libp2p, Metrics, TypedEventEmitter } from '@libp2p/interface' import type { DNS } from '@multiformats/dns' import type { Datastore } from 'interface-datastore' import type { Await } from 'interface-store' @@ -54,6 +54,11 @@ export interface Helia { */ datastore: Datastore + /** + * Event emitter for Helia start and stop events + */ + events: TypedEventEmitter> + /** * Pinning operations for blocks in the blockstore */ @@ -119,6 +124,30 @@ export interface GCOptions extends AbortOptions, ProgressOptions { } +export interface HeliaEvents { + /** + * This event notifies listeners that the node has started + * + * ```TypeScript + * helia.addEventListener('start', (event) => { + * console.info(event.detail.libp2p.isStarted()) // true + * }) + * ``` + */ + start: CustomEvent> + + /** + * This event notifies listeners that the node has stopped + * + * ```TypeScript + * helia.addEventListener('stop', (event) => { + * console.info(event.detail.libp2p.isStarted()) // false + * }) + * ``` + */ + stop: CustomEvent> +} + export * from './blocks.js' export * from './errors.js' export * from './pins.js' diff --git a/packages/interop/package.json b/packages/interop/package.json index ac63672a2..56fdb75a3 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "@helia/block-brokers": "^4.2.4", + "@helia/dnslink": "^0.0.0", "@helia/car": "^4.2.0", "@helia/dag-cbor": "^4.1.0", "@helia/dag-json": "^4.1.0", diff --git a/packages/interop/src/ipns-dnslink.spec.ts b/packages/interop/src/dnslink.spec.ts similarity index 75% rename from packages/interop/src/ipns-dnslink.spec.ts rename to packages/interop/src/dnslink.spec.ts index 482ea329c..f88bcd814 100644 --- a/packages/interop/src/ipns-dnslink.spec.ts +++ b/packages/interop/src/dnslink.spec.ts @@ -1,9 +1,9 @@ /* eslint-env mocha */ -import { ipns } from '@helia/ipns' +import { dnsLink } from '@helia/dnslink' import { expect } from 'aegir/chai' import { createHeliaNode } from './fixtures/create-helia.js' -import type { IPNS } from '@helia/ipns' +import type { DNSLink } from '@helia/dnslink' import type { DefaultLibp2pServices, Helia } from 'helia' import type { Libp2p } from 'libp2p' @@ -13,13 +13,13 @@ const TEST_DOMAINS: string[] = [ 'en.wikipedia-on-ipfs.org' ] -describe('@helia/ipns - dnslink', () => { +describe('@helia/dnslink', () => { let helia: Helia> - let name: IPNS + let name: DNSLink beforeEach(async () => { helia = await createHeliaNode() - name = ipns(helia) + name = dnsLink(helia) }) afterEach(async () => { @@ -30,7 +30,7 @@ describe('@helia/ipns - dnslink', () => { TEST_DOMAINS.forEach(domain => { it(`should resolve ${domain}`, async () => { - const result = await name.resolveDNSLink(domain) + const result = await name.resolve(domain) expect(result).to.have.property('cid') }).retries(5) diff --git a/packages/interop/src/ipns-pubsub.spec.ts b/packages/interop/src/ipns-pubsub.spec.ts index b564da1f3..616e781c3 100644 --- a/packages/interop/src/ipns-pubsub.spec.ts +++ b/packages/interop/src/ipns-pubsub.spec.ts @@ -72,6 +72,8 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { const cid = CID.createV1(raw.code, digest) const privateKey = await generateKeyPair('Ed25519') + const keyName = 'my-ipns-key' + await helia.libp2p.services.keychain.importKey(keyName, privateKey) // first call to pubsub resolver will fail but we should trigger // subscribing pubsub for updates @@ -104,7 +106,7 @@ keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { }) // publish should now succeed - await name.publish(privateKey, cid) + await name.publish(keyName, cid) // kubo should now be able to resolve IPNS name instantly const resolved = await last(kubo.api.name.resolve(privateKey.publicKey.toString(), { diff --git a/packages/interop/src/ipns.spec.ts b/packages/interop/src/ipns.spec.ts index 70b86b5b2..4cb8f0415 100644 --- a/packages/interop/src/ipns.spec.ts +++ b/packages/interop/src/ipns.spec.ts @@ -125,8 +125,9 @@ keyTypes.forEach(type => { await createNodes('kubo') const privateKey = await generateKeyPair('Ed25519') - - await name.publish(privateKey, value) + const keyName = 'my-ipns-key' + await helia.libp2p.services.keychain.importKey(keyName, privateKey) + await name.publish(keyName, value) const resolved = await last(kubo.api.name.resolve(privateKey.publicKey.toString())) diff --git a/packages/ipns/README.md b/packages/ipns/README.md index d4a21fe98..7c1e83c47 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -30,7 +30,7 @@ repo and examine the changes made. --> -IPNS operations using a Helia node +[IPNS](https://docs.ipfs.tech/concepts/ipns/) operations using a Helia node ## Example - Getting started @@ -40,23 +40,19 @@ With IPNSRouting routers: import { createHelia } from 'helia' import { ipns } from '@helia/ipns' import { unixfs } from '@helia/unixfs' -import { generateKeyPair } from '@libp2p/crypto/keys' const helia = await createHelia() const name = ipns(helia) -// create a keypair to publish an IPNS name -const privateKey = await generateKeyPair('Ed25519') - // store some data to publish const fs = unixfs(helia) const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) // publish the name -await name.publish(privateKey, cid) +const { publicKey } = await name.publish('key-1', cid) // resolve the name -const result = await name.resolve(privateKey.publicKey) +const result = await name.resolve(publicKey) console.info(result.cid, result.path) ``` @@ -75,24 +71,18 @@ import { generateKeyPair } from '@libp2p/crypto/keys' const helia = await createHelia() const name = ipns(helia) -// create a keypair to publish an IPNS name -const privateKey = await generateKeyPair('Ed25519') - // store some data to publish const fs = unixfs(helia) const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) // publish the name -await name.publish(privateKey, cid) - -// create another keypair to re-publish the original record -const recursivePrivateKey = await generateKeyPair('Ed25519') +const { publicKey } = await name.publish('key-1', cid) // publish the recursive name -await name.publish(recursivePrivateKey, privateKey.publicKey) +const { publicKey: recursivePublicKey } = await name.publish('key-2', publicKey) // resolve the name recursively - it resolves until a CID is found -const result = await name.resolve(recursivePrivateKey.publicKey) +const result = await name.resolve(recursivePublicKey) console.info(result.cid.toString() === cid.toString()) // true ``` @@ -109,9 +99,6 @@ import { generateKeyPair } from '@libp2p/crypto/keys' const helia = await createHelia() const name = ipns(helia) -// create a keypair to publish an IPNS name -const privateKey = await generateKeyPair('Ed25519') - // store some data to publish const fs = unixfs(helia) const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) @@ -121,10 +108,10 @@ const dirCid = await fs.addDirectory() const finalDirCid = await fs.cp(fileCid, dirCid, '/foo.txt') // publish the name -await name.publish(privateKey, `/ipfs/${finalDirCid}/foo.txt`) +const { publicKey } = await name.publish('key-1', `/ipfs/${finalDirCid}/foo.txt`) // resolve the name -const result = await name.resolve(privateKey.publicKey) +const result = await name.resolve(publicKey) console.info(result.cid, result.path) // QmFoo.. 'foo.txt' ``` @@ -168,133 +155,29 @@ const name = ipns(helia, { ] }) -// create a keypair to publish an IPNS name -const privateKey = await generateKeyPair('Ed25519') - // store some data to publish const fs = unixfs(helia) const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) // publish the name -await name.publish(privateKey, cid) +const { publicKey } = await name.publish('key-1', cid) // resolve the name -const result = await name.resolve(privateKey.publicKey) -``` - -## Example - Using custom DNS over HTTPS resolvers - -To use custom resolvers, configure Helia's `dns` option: - -```TypeScript -import { createHelia } from 'helia' -import { ipns } from '@helia/ipns' -import { dns } from '@multiformats/dns' -import { dnsOverHttps } from '@multiformats/dns/resolvers' -import { helia } from '@helia/ipns/routing' - -const node = await createHelia({ - dns: dns({ - resolvers: { - '.': dnsOverHttps('https://private-dns-server.me/dns-query') - } - }) -}) -const name = ipns(node, { - routers: [ - helia(node.routing) - ] -}) - -const result = name.resolveDNSLink('some-domain-with-dnslink-entry.com') -``` - -## Example - Resolving a domain with a dnslink entry - -Calling `resolveDNSLink` with the `@helia/ipns` instance: - -```TypeScript -// resolve a CID from a TXT record in a DNS zone file, using the default -// resolver for the current platform eg: -// > dig _dnslink.ipfs.io TXT -// ;; ANSWER SECTION: -// _dnslink.ipfs.io. 60 IN TXT "dnslink=/ipns/website.ipfs.io" -// > dig _dnslink.website.ipfs.io TXT -// ;; ANSWER SECTION: -// _dnslink.website.ipfs.io. 60 IN TXT "dnslink=/ipfs/QmWebsite" - -import { createHelia } from 'helia' -import { ipns } from '@helia/ipns' - -const node = await createHelia() -const name = ipns(node) - -const { answer } = await name.resolveDNSLink('ipfs.io') - -console.info(answer) -// { data: '/ipfs/QmWebsite' } -``` - -## Example - Using DNS-Over-HTTPS - -This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This -uses binary DNS records so requires extra dependencies to process the -response which can increase browser bundle sizes. - -If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. - -```TypeScript -import { createHelia } from 'helia' -import { ipns } from '@helia/ipns' -import { dns } from '@multiformats/dns' -import { dnsOverHttps } from '@multiformats/dns/resolvers' - -const node = await createHelia({ - dns: dns({ - resolvers: { - '.': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') - } - }) -}) -const name = ipns(node) - -const result = await name.resolveDNSLink('ipfs.io') -``` - -## Example - Using DNS-JSON-Over-HTTPS - -DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can -result in a smaller browser bundle due to the response being plain JSON. - -```TypeScript -import { createHelia } from 'helia' -import { ipns } from '@helia/ipns' -import { dns } from '@multiformats/dns' -import { dnsJsonOverHttps } from '@multiformats/dns/resolvers' - -const node = await createHelia({ - dns: dns({ - resolvers: { - '.': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') - } - }) -}) -const name = ipns(node) - -const result = await name.resolveDNSLink('ipfs.io') +const result = await name.resolve(publicKey) ``` ## Example - Republishing an existing IPNS record -The `republishRecord` method allows you to republish an existing IPNS record without -needing the private key. This is useful for relay nodes or when you want to extend -the availability of a record that was created elsewhere. +It is sometimes useful to be able to republish an existing IPNS record +without needing the private key. This allows you to extend the availability +of a record that was created elsewhere. ```TypeScript import { createHelia } from 'helia' -import { ipns } from '@helia/ipns' +import { ipns, ipnsValidator } from '@helia/ipns' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' import { CID } from 'multiformats/cid' +import { multihashToIPNSRoutingKey, marshalIPNSRecord } from 'ipns' const helia = await createHelia() const name = ipns(helia) @@ -304,7 +187,18 @@ const parsedCid: CID = CID.parse(ipnsName) const delegatedClient = createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev') const record = await delegatedClient.getIPNS(parsedCid) -await name.republishRecord(ipnsName, record) +const routingKey = multihashToIPNSRoutingKey(parsedCid.multihash) +const marshaledRecord = marshalIPNSRecord(record) + +// validate that they key corresponds to the record +await ipnsValidator(routingKey, marshaledRecord) + +// publish record to routing +await Promise.all( + name.routers.map(async r => { + await r.put(routingKey, marshaledRecord) + }) +) ``` # Install diff --git a/packages/ipns/package.json b/packages/ipns/package.json index c44ec65f6..fcab7191a 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -63,6 +63,7 @@ "doc-check": "aegir doc-check", "build": "aegir build", "docs": "aegir docs", + "generate": "protons ./src/pb/metadata.proto", "test": "aegir test", "test:chrome": "aegir test -t browser --cov", "test:chrome-webworker": "aegir test -t webworker", @@ -72,24 +73,27 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { + "@libp2p/crypto": "^5.1.7", "@helia/interface": "^5.4.0", "@libp2p/interface": "^3.0.2", "@libp2p/kad-dht": "^16.0.5", + "@libp2p/keychain": "^6.0.5", "@libp2p/logger": "^6.0.5", - "@libp2p/peer-id": "^6.0.3", - "@multiformats/dns": "^1.0.9", + "@libp2p/utils": "^7.0.5", "interface-datastore": "^9.0.2", "ipns": "^10.1.2", "multiformats": "^13.4.1", "progress-events": "^1.0.1", + "protons-runtime": "^5.5.0", + "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0" }, "devDependencies": { "@libp2p/crypto": "^5.1.12", - "@types/dns-packet": "^5.6.5", "aegir": "^47.0.22", "datastore-core": "^11.0.2", "it-drain": "^3.0.10", + "protons": "^7.6.1", "sinon": "^21.0.0", "sinon-ts": "^2.0.0" }, diff --git a/packages/ipns/src/constants.ts b/packages/ipns/src/constants.ts new file mode 100644 index 000000000..1b284b607 --- /dev/null +++ b/packages/ipns/src/constants.ts @@ -0,0 +1,24 @@ +const MINUTE = 60 * 1000 +const HOUR = 60 * MINUTE + +export const DEFAULT_LIFETIME_MS = 48 * HOUR + +/** + * The default DHT record expiry + */ +export const DHT_EXPIRY_MS = 48 * HOUR + +/** + * How often to run the republish loop + */ +export const DEFAULT_REPUBLISH_INTERVAL_MS = HOUR + +/** + * Republish IPNS records when the expiry of our provider records is within this + * threshold + */ +export const REPUBLISH_THRESHOLD = 24 * HOUR + +export const DEFAULT_TTL_NS = BigInt(MINUTE) * 5_000_000n // 5 minutes + +export const DEFAULT_REPUBLISH_CONCURRENCY = 5 diff --git a/packages/ipns/src/dnslink.ts b/packages/ipns/src/dnslink.ts deleted file mode 100644 index 48f34cabd..000000000 --- a/packages/ipns/src/dnslink.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { } from '@libp2p/interface' -import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id' -import { RecordType } from '@multiformats/dns' -import { CID } from 'multiformats/cid' -import { DNSLinkNotFoundError } from './errors.js' -import type { ResolveDNSLinkOptions } from './index.js' -import type { Logger, PeerId } from '@libp2p/interface' -import type { Answer, DNS } from '@multiformats/dns' - -const MAX_RECURSIVE_DEPTH = 32 - -export interface DNSLinkResult { - answer: Answer - value: string -} - -async function recursiveResolveDnslink (domain: string, depth: number, dns: DNS, log: Logger, options: ResolveDNSLinkOptions = {}): Promise { - if (depth === 0) { - throw new Error('recursion limit exceeded') - } - - log('query %s for TXT and CNAME records', domain) - const txtRecordsResponse = await dns.query(domain, { - ...options, - types: [ - RecordType.TXT - ] - }) - - // sort the TXT records to ensure deterministic processing - const txtRecords = (txtRecordsResponse?.Answer ?? []) - .sort((a, b) => a.data.localeCompare(b.data)) - - log('found %d TXT records for %s', txtRecords.length, domain) - - for (const answer of txtRecords) { - try { - let result = answer.data - - // strip leading and trailing " characters - if (result.startsWith('"') && result.endsWith('"')) { - result = result.substring(1, result.length - 1) - } - - if (!result.startsWith('dnslink=')) { - // invalid record? - continue - } - - log('%s TXT %s', answer.name, result) - - result = result.replace('dnslink=', '') - - // result is now a `/ipfs/` or `/ipns/` string - const [, protocol, domainOrCID, ...rest] = result.split('/') // e.g. ["", "ipfs", ""] - - if (protocol === 'ipfs') { - try { - const cid = CID.parse(domainOrCID) - - // if the result is a CID, we've reached the end of the recursion - return { - value: `/ipfs/${cid}${rest.length > 0 ? `/${rest.join('/')}` : ''}`, - answer - } - } catch {} - } else if (protocol === 'ipns') { - try { - let peerId: PeerId - - // eslint-disable-next-line max-depth - if (domainOrCID.charAt(0) === '1' || domainOrCID.charAt(0) === 'Q') { - peerId = peerIdFromString(domainOrCID) - } else { - // try parsing as a CID - peerId = peerIdFromCID(CID.parse(domainOrCID)) - } - - // if the result is a PeerId, we've reached the end of the recursion - return { - value: `/ipns/${peerId}${rest.length > 0 ? `/${rest.join('/')}` : ''}`, - answer - } - } catch {} - - // if the result was another IPNS domain, try to follow it - return await recursiveResolveDomain(domainOrCID, depth - 1, dns, log, options) - } else if (protocol === 'dnslink') { - // if the result was another DNSLink domain, try to follow it - return await recursiveResolveDomain(domainOrCID, depth - 1, dns, log, options) - } else { - log('unknown protocol "%s" in DNSLink record for domain: %s', protocol, domain) - continue - } - } catch (err: any) { - log.error('could not parse DNS link record for domain %s, %s', domain, answer.data, err) - } - } - - // no dnslink records found, try CNAMEs - log('no DNSLink records found for %s, falling back to CNAME', domain) - - const cnameRecordsResponse = await dns.query(domain, { - ...options, - types: [ - RecordType.CNAME - ] - }) - - // sort the CNAME records to ensure deterministic processing - const cnameRecords = (cnameRecordsResponse?.Answer ?? []) - .sort((a, b) => a.data.localeCompare(b.data)) - - log('found %d CNAME records for %s', cnameRecords.length, domain) - - for (const cname of cnameRecords) { - try { - return await recursiveResolveDomain(cname.data, depth - 1, dns, log, options) - } catch (err: any) { - log.error('domain %s cname %s had no DNSLink records', domain, cname.data, err) - } - } - - throw new DNSLinkNotFoundError(`No DNSLink records found for domain: ${domain}`) -} - -async function recursiveResolveDomain (domain: string, depth: number, dns: DNS, log: Logger, options: ResolveDNSLinkOptions = {}): Promise { - if (depth === 0) { - throw new Error('recursion limit exceeded') - } - - // the DNSLink spec says records MUST be stored on the `_dnslink.` subdomain - // so start looking for records there, we will fall back to the bare domain - // if none are found - if (!domain.startsWith('_dnslink.')) { - domain = `_dnslink.${domain}` - } - - try { - return await recursiveResolveDnslink(domain, depth, dns, log, options) - } catch (err: any) { - // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error - if (err.code !== 'ENOTFOUND' && err.code !== 'ENODATA' && err.name !== 'DNSLinkNotFoundError' && err.name !== 'NotFoundError') { - throw err - } - - if (domain.startsWith('_dnslink.')) { - // The supplied domain contains a _dnslink component - // Check the non-_dnslink domain - domain = domain.replace('_dnslink.', '') - } else { - // Check the _dnslink subdomain - domain = `_dnslink.${domain}` - } - - // If this throws then we propagate the error - return recursiveResolveDnslink(domain, depth, dns, log, options) - } -} - -export async function resolveDNSLink (domain: string, dns: DNS, log: Logger, options: ResolveDNSLinkOptions = {}): Promise { - return recursiveResolveDomain(domain, options.maxRecursiveDepth ?? MAX_RECURSIVE_DEPTH, dns, log, options) -} diff --git a/packages/ipns/src/errors.ts b/packages/ipns/src/errors.ts index 39360163d..f43e2cf7c 100644 --- a/packages/ipns/src/errors.ts +++ b/packages/ipns/src/errors.ts @@ -1,12 +1,3 @@ -export class DNSLinkNotFoundError extends Error { - static name = 'DNSLinkNotFoundError' - - constructor (message = 'DNSLink not found') { - super(message) - this.name = 'DNSLinkNotFoundError' - } -} - export class RecordsFailedValidationError extends Error { static name = 'RecordsFailedValidationError' diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 85e34cd56..aaca2b97c 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -1,7 +1,7 @@ /** * @packageDocumentation * - * IPNS operations using a Helia node + * [IPNS](https://docs.ipfs.tech/concepts/ipns/) operations using a Helia node * * @example Getting started * @@ -11,23 +11,19 @@ * import { createHelia } from 'helia' * import { ipns } from '@helia/ipns' * import { unixfs } from '@helia/unixfs' - * import { generateKeyPair } from '@libp2p/crypto/keys' * * const helia = await createHelia() * const name = ipns(helia) * - * // create a keypair to publish an IPNS name - * const privateKey = await generateKeyPair('Ed25519') - * * // store some data to publish * const fs = unixfs(helia) * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) * * // publish the name - * await name.publish(privateKey, cid) + * const { publicKey } = await name.publish('key-1', cid) * * // resolve the name - * const result = await name.resolve(privateKey.publicKey) + * const result = await name.resolve(publicKey) * * console.info(result.cid, result.path) * ``` @@ -46,24 +42,18 @@ * const helia = await createHelia() * const name = ipns(helia) * - * // create a keypair to publish an IPNS name - * const privateKey = await generateKeyPair('Ed25519') - * * // store some data to publish * const fs = unixfs(helia) * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) * * // publish the name - * await name.publish(privateKey, cid) - * - * // create another keypair to re-publish the original record - * const recursivePrivateKey = await generateKeyPair('Ed25519') + * const { publicKey } = await name.publish('key-1', cid) * * // publish the recursive name - * await name.publish(recursivePrivateKey, privateKey.publicKey) + * const { publicKey: recursivePublicKey } = await name.publish('key-2', publicKey) * * // resolve the name recursively - it resolves until a CID is found - * const result = await name.resolve(recursivePrivateKey.publicKey) + * const result = await name.resolve(recursivePublicKey) * console.info(result.cid.toString() === cid.toString()) // true * ``` * @@ -80,9 +70,6 @@ * const helia = await createHelia() * const name = ipns(helia) * - * // create a keypair to publish an IPNS name - * const privateKey = await generateKeyPair('Ed25519') - * * // store some data to publish * const fs = unixfs(helia) * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) @@ -92,10 +79,10 @@ * const finalDirCid = await fs.cp(fileCid, dirCid, '/foo.txt') * * // publish the name - * await name.publish(privateKey, `/ipfs/${finalDirCid}/foo.txt`) + * const { publicKey } = await name.publish('key-1', `/ipfs/${finalDirCid}/foo.txt`) * * // resolve the name - * const result = await name.resolve(privateKey.publicKey) + * const result = await name.resolve(publicKey) * * console.info(result.cid, result.path) // QmFoo.. 'foo.txt' * ``` @@ -139,133 +126,30 @@ * ] * }) * - * // create a keypair to publish an IPNS name - * const privateKey = await generateKeyPair('Ed25519') * * // store some data to publish * const fs = unixfs(helia) * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) * * // publish the name - * await name.publish(privateKey, cid) + * const { publicKey } = await name.publish('key-1', cid) * * // resolve the name - * const result = await name.resolve(privateKey.publicKey) - * ``` - * - * @example Using custom DNS over HTTPS resolvers - * - * To use custom resolvers, configure Helia's `dns` option: - * - * ```TypeScript - * import { createHelia } from 'helia' - * import { ipns } from '@helia/ipns' - * import { dns } from '@multiformats/dns' - * import { dnsOverHttps } from '@multiformats/dns/resolvers' - * import { helia } from '@helia/ipns/routing' - * - * const node = await createHelia({ - * dns: dns({ - * resolvers: { - * '.': dnsOverHttps('https://private-dns-server.me/dns-query') - * } - * }) - * }) - * const name = ipns(node, { - * routers: [ - * helia(node.routing) - * ] - * }) - * - * const result = name.resolveDNSLink('some-domain-with-dnslink-entry.com') - * ``` - * - * @example Resolving a domain with a dnslink entry - * - * Calling `resolveDNSLink` with the `@helia/ipns` instance: - * - * ```TypeScript - * // resolve a CID from a TXT record in a DNS zone file, using the default - * // resolver for the current platform eg: - * // > dig _dnslink.ipfs.io TXT - * // ;; ANSWER SECTION: - * // _dnslink.ipfs.io. 60 IN TXT "dnslink=/ipns/website.ipfs.io" - * // > dig _dnslink.website.ipfs.io TXT - * // ;; ANSWER SECTION: - * // _dnslink.website.ipfs.io. 60 IN TXT "dnslink=/ipfs/QmWebsite" - * - * import { createHelia } from 'helia' - * import { ipns } from '@helia/ipns' - * - * const node = await createHelia() - * const name = ipns(node) - * - * const { answer } = await name.resolveDNSLink('ipfs.io') - * - * console.info(answer) - * // { data: '/ipfs/QmWebsite' } - * ``` - * - * @example Using DNS-Over-HTTPS - * - * This example uses the Mozilla provided RFC 1035 DNS over HTTPS service. This - * uses binary DNS records so requires extra dependencies to process the - * response which can increase browser bundle sizes. - * - * If this is a concern, use the DNS-JSON-Over-HTTPS resolver instead. - * - * ```TypeScript - * import { createHelia } from 'helia' - * import { ipns } from '@helia/ipns' - * import { dns } from '@multiformats/dns' - * import { dnsOverHttps } from '@multiformats/dns/resolvers' - * - * const node = await createHelia({ - * dns: dns({ - * resolvers: { - * '.': dnsOverHttps('https://mozilla.cloudflare-dns.com/dns-query') - * } - * }) - * }) - * const name = ipns(node) - * - * const result = await name.resolveDNSLink('ipfs.io') - * ``` - * - * @example Using DNS-JSON-Over-HTTPS - * - * DNS-JSON-Over-HTTPS resolvers use the RFC 8427 `application/dns-json` and can - * result in a smaller browser bundle due to the response being plain JSON. - * - * ```TypeScript - * import { createHelia } from 'helia' - * import { ipns } from '@helia/ipns' - * import { dns } from '@multiformats/dns' - * import { dnsJsonOverHttps } from '@multiformats/dns/resolvers' - * - * const node = await createHelia({ - * dns: dns({ - * resolvers: { - * '.': dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query') - * } - * }) - * }) - * const name = ipns(node) - * - * const result = await name.resolveDNSLink('ipfs.io') + * const result = await name.resolve(publicKey) * ``` * * @example Republishing an existing IPNS record * - * The `republishRecord` method allows you to republish an existing IPNS record without - * needing the private key. This is useful for relay nodes or when you want to extend - * the availability of a record that was created elsewhere. + * It is sometimes useful to be able to republish an existing IPNS record + * without needing the private key. This allows you to extend the availability + * of a record that was created elsewhere. * * ```TypeScript * import { createHelia } from 'helia' - * import { ipns } from '@helia/ipns' + * import { ipns, ipnsValidator } from '@helia/ipns' * import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' * import { CID } from 'multiformats/cid' + * import { multihashToIPNSRoutingKey, marshalIPNSRecord } from 'ipns' * * const helia = await createHelia() * const name = ipns(helia) @@ -275,47 +159,33 @@ * const delegatedClient = createDelegatedRoutingV1HttpApiClient('https://delegated-ipfs.dev') * const record = await delegatedClient.getIPNS(parsedCid) * - * await name.republishRecord(ipnsName, record) + * const routingKey = multihashToIPNSRoutingKey(parsedCid.multihash) + * const marshaledRecord = marshalIPNSRecord(record) + * + * // validate that they key corresponds to the record + * await ipnsValidator(routingKey, marshaledRecord) + * + * // publish record to routing + * await Promise.all( + * name.routers.map(async r => { + * await r.put(routingKey, marshaledRecord) + * }) + * ) * ``` */ -import { NotFoundError, isPublicKey } from '@libp2p/interface' -import { logger } from '@libp2p/logger' -import { peerIdFromString } from '@libp2p/peer-id' -import { createIPNSRecord, extractPublicKeyFromIPNSRecord, marshalIPNSRecord, multihashToIPNSRoutingKey, unmarshalIPNSRecord } from 'ipns' -import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' -import { base36 } from 'multiformats/bases/base36' -import { base58btc } from 'multiformats/bases/base58' import { CID } from 'multiformats/cid' -import * as Digest from 'multiformats/hashes/digest' -import { CustomProgressEvent } from 'progress-events' -import { resolveDNSLink } from './dnslink.js' -import { InvalidValueError, RecordsFailedValidationError, UnsupportedMultibasePrefixError, UnsupportedMultihashCodecError } from './errors.js' -import { helia } from './routing/helia.js' -import { localStore } from './routing/local-store.js' -import { isCodec, IDENTITY_CODEC, SHA2_256_CODEC, IPNS_STRING_PREFIX } from './utils.js' +import { IPNS as IPNSClass } from './ipns.js' import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' -import type { LocalStore } from './routing/local-store.js' -import type { Routing } from '@helia/interface' -import type { AbortOptions, ComponentLogger, Logger, PrivateKey, PublicKey } from '@libp2p/interface' -import type { Answer, DNS, ResolveDnsProgressEvents } from '@multiformats/dns' +import type { Routing, HeliaEvents } from '@helia/interface' +import type { AbortOptions, ComponentLogger, Libp2p, PublicKey, TypedEventEmitter } from '@libp2p/interface' +import type { Keychain } from '@libp2p/keychain' import type { Datastore } from 'interface-datastore' import type { IPNSRecord } from 'ipns' -import type { MultibaseDecoder } from 'multiformats/bases/interface' import type { MultihashDigest } from 'multiformats/hashes/interface' import type { ProgressEvent, ProgressOptions } from 'progress-events' -const log = logger('helia:ipns') - -const MINUTE = 60 * 1000 -const HOUR = 60 * MINUTE - -const DEFAULT_LIFETIME_MS = 48 * HOUR -const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR - -const DEFAULT_TTL_NS = BigInt(MINUTE) * 5_000_000n // 5 minutes - export type PublishProgressEvents = ProgressEvent<'ipns:publish:start'> | ProgressEvent<'ipns:publish:success', IPNSRecord> | @@ -326,15 +196,11 @@ export type ResolveProgressEvents = ProgressEvent<'ipns:resolve:success', IPNSRecord> | ProgressEvent<'ipns:resolve:error', Error> -export type RepublishProgressEvents = - ProgressEvent<'ipns:republish:start', unknown> | - ProgressEvent<'ipns:republish:success', IPNSRecord> | - ProgressEvent<'ipns:republish:error', { key?: MultihashDigest<0x00 | 0x12>, record: IPNSRecord, err: Error }> - -export type ResolveDNSLinkProgressEvents = - ResolveProgressEvents | - IPNSRoutingEvents | - ResolveDnsProgressEvents +export type DatastoreProgressEvents = + ProgressEvent<'ipns:routing:datastore:put'> | + ProgressEvent<'ipns:routing:datastore:get'> | + ProgressEvent<'ipns:routing:datastore:list'> | + ProgressEvent<'ipns:routing:datastore:error', Error> export interface PublishOptions extends AbortOptions, ProgressOptions { /** @@ -359,23 +225,12 @@ export interface PublishOptions extends AbortOptions, ProgressOptions { - /** - * Do not query the network for the IPNS record - * - * @default false - */ - offline?: boolean - - /** - * Do not use cached IPNS Record entries - * - * @default false - */ - nocache?: boolean +export interface IPNSRecordMetadata { + keyName: string + lifetime: number } -export interface ResolveDNSLinkOptions extends AbortOptions, ProgressOptions { +export interface ResolveOptions extends AbortOptions, ProgressOptions { /** * Do not query the network for the IPNS record * @@ -384,35 +239,11 @@ export interface ResolveDNSLinkOptions extends AbortOptions, ProgressOptions { - /** - * The republish interval in ms (default: 23hrs) - */ - interval?: number -} - -export interface RepublishRecordOptions extends AbortOptions, ProgressOptions { - /** - * Only publish to a local datastore - * - * @default false - */ - offline?: boolean } export interface ResolveResult { @@ -436,44 +267,64 @@ export interface IPNSResolveResult extends ResolveResult { record: IPNSRecord } -export interface DNSLinkResolveResult extends ResolveResult { +export interface IPNSPublishResult { /** - * The resolved record + * The published record */ - answer: Answer -} + record: IPNSRecord -export interface IPNS { /** - * Creates an IPNS record signed by the passed PeerId that will resolve to the passed value - * - * If the value is a PeerId, a recursive IPNS record will be created. + * The public key that was used to publish the record */ - publish(key: PrivateKey, value: CID | PublicKey | MultihashDigest<0x00 | 0x12> | string, options?: PublishOptions): Promise + publicKey: PublicKey +} +export interface IPNS { /** - * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record - * corresponding to that public key until a value is found + * Configured routing subsystems used to publish/resolve IPNS names */ - resolve(key: PublicKey | MultihashDigest<0x00 | 0x12>, options?: ResolveOptions): Promise + routers: IPNSRouting[] /** - * Resolve a CID from a dns-link style IPNS record + * Creates an IPNS record signed by the passed PeerId that will resolve to the + * passed value + * + * If the value is a PeerId, a recursive IPNS record will be created. + * + * @example + * + * ```TypeScript + * import { createHelia } from 'helia' + * import { ipns } from '@helia/ipns' + * + * const helia = await createHelia() + * const name = ipns(helia) + * + * const result = await name.publish('my-key-name', cid, { + * signal: AbortSignal.timeout(5_000) + * }) + * + * console.info(result) // { answer: ... } + * ``` */ - resolveDNSLink(domain: string, options?: ResolveDNSLinkOptions): Promise + publish(keyName: string, value: CID | PublicKey | MultihashDigest<0x00 | 0x12> | string, options?: PublishOptions): Promise /** - * Periodically republish all IPNS records found in the datastore + * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS + * record corresponding to that public key until a value is found */ - republish(options?: RepublishOptions): void + resolve(key: PublicKey | MultihashDigest<0x00 | 0x12>, options?: ResolveOptions): Promise /** - * Republish an existing IPNS record without the private key. + * Stop republishing of an IPNS record + * + * This will delete the last signed IPNS record from the datastore, but the + * key will remain in the keychain. * - * Before republishing the record will be validated to ensure it has a valid signature and lifetime(validity) in the future. - * The key is a multihash of the public key or a string representation of the PeerID (either base58btc encoded multihash or base36 encoded CID) + * Note that the record may still be resolved by other peers until it expires + * or is no longer valid. */ - republishRecord(key: MultihashDigest<0x00 | 0x12> | string, record: IPNSRecord, options?: RepublishRecordOptions): Promise + unpublish(keyName: string, options?: AbortOptions): Promise } export type { IPNSRouting } from './routing/index.js' @@ -483,325 +334,35 @@ export type { IPNSRecord } from 'ipns' export interface IPNSComponents { datastore: Datastore routing: Routing - dns: DNS logger: ComponentLogger -} - -const bases: Record> = { - [base36.prefix]: base36, - [base58btc.prefix]: base58btc -} - -class DefaultIPNS implements IPNS { - private readonly routers: IPNSRouting[] - private readonly localStore: LocalStore - private timeout?: ReturnType - private readonly dns: DNS - private readonly log: Logger - - constructor (components: IPNSComponents, routers: IPNSRouting[] = []) { - this.routers = [ - helia(components.routing), - ...routers - ] - this.localStore = localStore(components.datastore) - this.dns = components.dns - this.log = components.logger.forComponent('helia:ipns') - } - - async publish (key: PrivateKey, value: CID | PublicKey | MultihashDigest<0x00 | 0x12> | string, options: PublishOptions = {}): Promise { - try { - let sequenceNumber = 1n - const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) - - if (await this.localStore.has(routingKey, options)) { - // if we have published under this key before, increment the sequence number - const { record } = await this.localStore.get(routingKey, options) - const existingRecord = unmarshalIPNSRecord(record) - sequenceNumber = existingRecord.sequence + 1n - } - - // convert ttl from milliseconds to nanoseconds as createIPNSRecord expects - const ttlNs = options.ttl != null ? BigInt(options.ttl) * 1_000_000n : DEFAULT_TTL_NS - const record = await createIPNSRecord(key, value, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS, { ...options, ttlNs }) - const marshaledRecord = marshalIPNSRecord(record) - - await this.localStore.put(routingKey, marshaledRecord, options) - - if (options.offline !== true) { - // publish record to routing - await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) - } - - return record - } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:publish:error', err)) - throw err - } - } - - async resolve (key: PublicKey | MultihashDigest<0x00 | 0x12>, options: ResolveOptions = {}): Promise { - const digest = isPublicKey(key) ? key.toMultihash() : key - const routingKey = multihashToIPNSRoutingKey(digest) - const record = await this.#findIpnsRecord(routingKey, options) - - return { - ...(await this.#resolve(record.value, options)), - record - } - } - - async resolveDNSLink (domain: string, options: ResolveDNSLinkOptions = {}): Promise { - const dnslink = await resolveDNSLink(domain, this.dns, this.log, options) - - return { - ...(await this.#resolve(dnslink.value, options)), - answer: dnslink.answer - } - } - - republish (options: RepublishOptions = {}): void { - if (this.timeout != null) { - throw new Error('Republish is already running') - } - - options.signal?.addEventListener('abort', () => { - clearTimeout(this.timeout) - }) - - async function republish (): Promise { - const startTime = Date.now() - - options.onProgress?.(new CustomProgressEvent('ipns:republish:start')) - - const finishType = Date.now() - const timeTaken = finishType - startTime - let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken - - if (nextInterval < 0) { - nextInterval = options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS - } - - setTimeout(() => { - republish().catch(err => { - log.error('error republishing', err) - }) - }, nextInterval) - } - - this.timeout = setTimeout(() => { - republish().catch(err => { - log.error('error republishing', err) - }) - }, options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS) - } - - async #resolve (ipfsPath: string, options: ResolveOptions = {}): Promise<{ cid: CID, path: string }> { - const parts = ipfsPath.split('/') - try { - const scheme = parts[1] - - if (scheme === 'ipns') { - const str = parts[2] - const prefix = str.substring(0, 1) - let buf: Uint8Array | undefined - - if (prefix === '1' || prefix === 'Q') { - buf = base58btc.decode(`z${str}`) - } else if (bases[prefix] != null) { - buf = bases[prefix].decode(str) - } else { - throw new UnsupportedMultibasePrefixError(`Unsupported multibase prefix "${prefix}"`) - } - - let digest: MultihashDigest - - try { - digest = Digest.decode(buf) - } catch { - digest = CID.decode(buf).multihash - } - - if (!isCodec(digest, IDENTITY_CODEC) && !isCodec(digest, SHA2_256_CODEC)) { - throw new UnsupportedMultihashCodecError(`Unsupported multihash codec "${digest.code}"`) - } - - const { cid } = await this.resolve(digest, options) - const path = parts.slice(3).join('/') - return { - cid, - path - } - } else if (scheme === 'ipfs') { - const cid = CID.parse(parts[2]) - const path = parts.slice(3).join('/') - return { - cid, - path - } - } - } catch (err) { - log.error('error parsing ipfs path', err) - } - - log.error('invalid ipfs path %s', ipfsPath) - throw new InvalidValueError('Invalid value') - } - - async #findIpnsRecord (routingKey: Uint8Array, options: ResolveOptions = {}): Promise { - const records: Uint8Array[] = [] - const cached = await this.localStore.has(routingKey, options) - - if (cached) { - log('record is present in the cache') - - if (options.nocache !== true) { - try { - // check the local cache first - const { record, created } = await this.localStore.get(routingKey, options) - - this.log('record retrieved from cache') - - // validate the record - await ipnsValidator(routingKey, record) - - this.log('record was valid') - - // check the TTL - const ipnsRecord = unmarshalIPNSRecord(record) - - // IPNS TTL is in nanoseconds, convert to milliseconds, default to one - // hour - const ttlMs = Number((ipnsRecord.ttl ?? DEFAULT_TTL_NS) / 1_000_000n) - const ttlExpires = created.getTime() + ttlMs - - if (ttlExpires > Date.now()) { - // the TTL has not yet expired, return the cached record - this.log('record TTL was valid') - return ipnsRecord - } - - if (options.offline === true) { - // the TTL has expired but we are skipping the routing search - this.log('record TTL has been reached but we are resolving offline-only, returning record') - return ipnsRecord - } - - this.log('record TTL has been reached, searching routing for updates') - - // add the local record to our list of resolved record, and also - // search the routing for updates - the most up to date record will be - // returned - records.push(record) - } catch (err) { - this.log('cached record was invalid', err) - await this.localStore.delete(routingKey, options) - } - } else { - log('ignoring local cache due to nocache=true option') - } - } - - if (options.offline === true) { - throw new NotFoundError('Record was not present in the cache or has expired') - } - - log('did not have record locally') - - let foundInvalid = 0 - - await Promise.all( - this.routers.map(async (router) => { - let record: Uint8Array - - try { - record = await router.get(routingKey, { - ...options, - validate: false - }) - } catch (err: any) { - log.error('error finding IPNS record', err) - - return - } - - try { - await ipnsValidator(routingKey, record) - - records.push(record) - } catch (err) { - // we found a record, but the validator rejected it - foundInvalid++ - log.error('error finding IPNS record', err) - } - }) - ) - - if (records.length === 0) { - if (foundInvalid > 0) { - throw new RecordsFailedValidationError(`${foundInvalid > 1 ? `${foundInvalid} records` : 'Record'} found for routing key ${foundInvalid > 1 ? 'were' : 'was'} invalid`) - } - - throw new NotFoundError('Could not find record for routing key') - } - - const record = records[ipnsSelector(routingKey, records)] - - await this.localStore.put(routingKey, record, options) - - return unmarshalIPNSRecord(record) - } - - async republishRecord (key: MultihashDigest<0x00 | 0x12> | string, record: IPNSRecord, options: RepublishRecordOptions = {}): Promise { - let mh: MultihashDigest<0x00 | 0x12> | undefined - try { - mh = extractPublicKeyFromIPNSRecord(record)?.toMultihash() // embedded public key take precedence, if present - if (mh == null) { - // if no public key is embedded in the record, use the key that was passed in - if (typeof key === 'string') { - if (key.startsWith(IPNS_STRING_PREFIX)) { - // remove the /ipns/ prefix from the key - key = key.slice(IPNS_STRING_PREFIX.length) - } - // Convert string key to MultihashDigest - try { - mh = peerIdFromString(key).toMultihash() - } catch (err: any) { - throw new Error(`Invalid string key: ${err.message}`) - } - } else { - mh = key - } - } - - if (mh == null) { - throw new Error('No public key multihash found to determine the routing key') - } - - const routingKey = multihashToIPNSRoutingKey(mh) - const marshaledRecord = marshalIPNSRecord(record) - - await ipnsValidator(routingKey, marshaledRecord) // validate that they key corresponds to the record - - await this.localStore.put(routingKey, marshaledRecord, options) // add to local store - - if (options.offline !== true) { - // publish record to routing - await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) - } - } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:republish:error', { key: mh, record, err })) - throw err - } - } + libp2p: Libp2p<{ keychain: Keychain }> + events: TypedEventEmitter // Helia event bus } export interface IPNSOptions { + /** + * Different routing systems for IPNS publishing/resolving + */ routers?: IPNSRouting[] + + /** + * How often to check if published records have expired and need republishing + * in ms + * + * @default 3_600_000 + */ + republishInterval?: number + + /** + * How many IPNS records to republish at once + * + * @default 5 + */ + republishConcurrency?: number } -export function ipns (components: IPNSComponents, { routers = [] }: IPNSOptions = {}): IPNS { - return new DefaultIPNS(components, routers) +export function ipns (components: IPNSComponents, options: IPNSOptions = {}): IPNS { + return new IPNSClass(components, options) } export { ipnsValidator, type IPNSRoutingEvents } diff --git a/packages/ipns/src/ipns.ts b/packages/ipns/src/ipns.ts new file mode 100644 index 000000000..815cee381 --- /dev/null +++ b/packages/ipns/src/ipns.ts @@ -0,0 +1,396 @@ +import { generateKeyPair } from '@libp2p/crypto/keys' +import { NotFoundError, NotStartedError, isPublicKey } from '@libp2p/interface' +import { Queue, repeatingTask } from '@libp2p/utils' +import { createIPNSRecord, marshalIPNSRecord, multihashToIPNSRoutingKey, unmarshalIPNSRecord } from 'ipns' +import { ipnsSelector } from 'ipns/selector' +import { ipnsValidator } from 'ipns/validator' +import { base36 } from 'multiformats/bases/base36' +import { base58btc } from 'multiformats/bases/base58' +import { CID } from 'multiformats/cid' +import * as Digest from 'multiformats/hashes/digest' +import { CustomProgressEvent } from 'progress-events' +import { DEFAULT_LIFETIME_MS, DEFAULT_REPUBLISH_CONCURRENCY, DEFAULT_REPUBLISH_INTERVAL_MS, DEFAULT_TTL_NS } from './constants.ts' +import { InvalidValueError, RecordsFailedValidationError, UnsupportedMultibasePrefixError, UnsupportedMultihashCodecError } from './errors.js' +import { localStore } from './local-store.js' +import { helia } from './routing/helia.js' +import { localStoreRouting } from './routing/local-store.ts' +import { isCodec, IDENTITY_CODEC, SHA2_256_CODEC, shouldRepublish } from './utils.js' +import type { IPNSComponents, IPNS as IPNSInterface, IPNSOptions, IPNSPublishResult, IPNSResolveResult, PublishOptions, ResolveOptions } from './index.js' +import type { LocalStore } from './local-store.js' +import type { IPNSRouting } from './routing/index.js' +import type { AbortOptions, Logger, PrivateKey, PublicKey, Startable } from '@libp2p/interface' +import type { Keychain } from '@libp2p/keychain' +import type { RepeatingTask } from '@libp2p/utils' +import type { IPNSRecord } from 'ipns' +import type { MultibaseDecoder } from 'multiformats/bases/interface' +import type { MultihashDigest } from 'multiformats/hashes/interface' + +const bases: Record> = { + [base36.prefix]: base36, + [base58btc.prefix]: base58btc +} + +export class IPNS implements IPNSInterface, Startable { + public readonly routers: IPNSRouting[] + private readonly localStore: LocalStore + private readonly republishTask: RepeatingTask + private readonly log: Logger + private readonly keychain: Keychain + private started: boolean = false + private readonly republishConcurrency: number + + constructor (components: IPNSComponents, init: IPNSOptions = {}) { + this.log = components.logger.forComponent('helia:ipns') + this.localStore = localStore(components.datastore, components.logger.forComponent('helia:ipns:local-store')) + this.keychain = components.libp2p.services.keychain + this.republishConcurrency = init.republishConcurrency || DEFAULT_REPUBLISH_CONCURRENCY + this.started = components.libp2p.status === 'started' + + this.routers = [ + localStoreRouting(this.localStore), + helia(components.routing), + ...(init.routers ?? []) + ] + + // start republishing on Helia start + components.events.addEventListener('start', this.start.bind(this)) + // stop republishing on Helia stop + components.events.addEventListener('stop', this.stop.bind(this)) + + this.republishTask = repeatingTask(this.#republish.bind(this), init.republishInterval ?? DEFAULT_REPUBLISH_INTERVAL_MS, { + runImmediately: true + }) + + if (this.started) { + this.republishTask.start() + } + } + + start (): void { + if (this.started) { + return + } + + this.started = true + this.republishTask.start() + } + + stop (): void { + if (!this.started) { + return + } + + this.started = false + this.republishTask.stop() + } + + #throwIfStopped (): void { + if (!this.started) { + throw new NotStartedError('Helia is stopped, cannot perform IPNS operations') + } + } + + async publish (keyName: string, value: CID | PublicKey | MultihashDigest<0x00 | 0x12> | string, options: PublishOptions = {}): Promise { + this.#throwIfStopped() + + try { + const privKey = await this.#loadOrCreateKey(keyName) + let sequenceNumber = 1n + const routingKey = multihashToIPNSRoutingKey(privKey.publicKey.toMultihash()) + + if (await this.localStore.has(routingKey, options)) { + // if we have published under this key before, increment the sequence number + const { record } = await this.localStore.get(routingKey, options) + const existingRecord = unmarshalIPNSRecord(record) + sequenceNumber = existingRecord.sequence + 1n + } + + // convert ttl from milliseconds to nanoseconds as createIPNSRecord expects + const ttlNs = options.ttl != null ? BigInt(options.ttl) * 1_000_000n : DEFAULT_TTL_NS + const lifetime = options.lifetime ?? DEFAULT_LIFETIME_MS + const record = await createIPNSRecord(privKey, value, sequenceNumber, lifetime, { ...options, ttlNs }) + const marshaledRecord = marshalIPNSRecord(record) + + if (options.offline === true) { + // only store record locally + await this.localStore.put(routingKey, marshaledRecord, { ...options, metadata: { keyName, lifetime } }) + } else { + // publish record to routing (including the local store) + await Promise.all(this.routers.map(async r => { + await r.put(routingKey, marshaledRecord, { ...options, metadata: { keyName, lifetime } }) + })) + } + + return { + record, + publicKey: privKey.publicKey + } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:publish:error', err)) + throw err + } + } + + async resolve (key: PublicKey | MultihashDigest<0x00 | 0x12>, options: ResolveOptions = {}): Promise { + this.#throwIfStopped() + const digest = isPublicKey(key) ? key.toMultihash() : key + const routingKey = multihashToIPNSRoutingKey(digest) + const record = await this.#findIpnsRecord(routingKey, options) + + return { + ...(await this.#resolve(record.value, options)), + record + } + } + + async #loadOrCreateKey (keyName: string): Promise { + try { + return await this.keychain.exportKey(keyName) + } catch (err: any) { + // If no named key found in keychain, generate and import + const privKey = await generateKeyPair('Ed25519') + await this.keychain.importKey(keyName, privKey) + return privKey + } + } + + async #republish (options: AbortOptions = {}): Promise { + if (!this.started) { + return + } + + this.log('starting ipns republish records loop') + + const queue = new Queue({ + concurrency: this.republishConcurrency + }) + + try { + const recordsToRepublish: Array<{ routingKey: Uint8Array, record: IPNSRecord }> = [] + + // Find all records using the localStore.list method + for await (const { routingKey, record, metadata, created } of this.localStore.list(options)) { + if (metadata == null) { + // Skip if no metadata is found from before we started + // storing metadata or for records republished without a key + this.log(`no metadata found for record ${routingKey.toString()}, skipping`) + continue + } + let ipnsRecord: IPNSRecord + try { + ipnsRecord = unmarshalIPNSRecord(record) + } catch (err: any) { + this.log.error('error unmarshaling record - %e', err) + continue + } + + // Only republish records that are within the DHT or record expiry threshold + if (!shouldRepublish(ipnsRecord, created)) { + this.log.trace(`skipping record ${routingKey.toString()}within republish threshold`) + continue + } + const sequenceNumber = ipnsRecord.sequence + 1n + const ttlNs = ipnsRecord.ttl ?? DEFAULT_TTL_NS + let privKey: PrivateKey + + try { + privKey = await this.keychain.exportKey(metadata.keyName) + } catch (err: any) { + this.log.error(`missing key ${metadata.keyName}, skipping republishing record`, err) + continue + } + try { + const updatedRecord = await createIPNSRecord(privKey, ipnsRecord.value, sequenceNumber, metadata.lifetime, { ...options, ttlNs }) + recordsToRepublish.push({ routingKey, record: updatedRecord }) + } catch (err: any) { + this.log.error(`error creating updated IPNS record for ${routingKey.toString()}`, err) + continue + } + } + + this.log(`found ${recordsToRepublish.length} records to republish`) + + // Republish each record + for (const { routingKey, record } of recordsToRepublish) { + // Add job to queue to republish the record to all routers + queue.add(async () => { + try { + const marshaledRecord = marshalIPNSRecord(record) + await Promise.all( + this.routers.map(r => r.put(routingKey, marshaledRecord, options)) + ) + } catch (err: any) { + this.log.error('error republishing record - %e', err) + } + }, options) + } + } catch (err: any) { + this.log.error('error during republish - %e', err) + } + + await queue.onIdle(options) // Wait for all jobs to complete + } + + async unpublish (keyName: string, options?: AbortOptions): Promise { + const { publicKey } = await this.keychain.exportKey(keyName) + const digest = publicKey.toMultihash() + const routingKey = multihashToIPNSRoutingKey(digest) + await this.localStore.delete(routingKey, options) + } + + async #resolve (ipfsPath: string, options: ResolveOptions = {}): Promise<{ cid: CID, path: string }> { + const parts = ipfsPath.split('/') + try { + const scheme = parts[1] + + if (scheme === 'ipns') { + const str = parts[2] + const prefix = str.substring(0, 1) + let buf: Uint8Array | undefined + + if (prefix === '1' || prefix === 'Q') { + buf = base58btc.decode(`z${str}`) + } else if (bases[prefix] != null) { + buf = bases[prefix].decode(str) + } else { + throw new UnsupportedMultibasePrefixError(`Unsupported multibase prefix "${prefix}"`) + } + + let digest: MultihashDigest + + try { + digest = Digest.decode(buf) + } catch { + digest = CID.decode(buf).multihash + } + + if (!isCodec(digest, IDENTITY_CODEC) && !isCodec(digest, SHA2_256_CODEC)) { + throw new UnsupportedMultihashCodecError(`Unsupported multihash codec "${digest.code}"`) + } + + const { cid } = await this.resolve(digest, options) + const path = parts.slice(3).join('/') + return { + cid, + path + } + } else if (scheme === 'ipfs') { + const cid = CID.parse(parts[2]) + const path = parts.slice(3).join('/') + return { + cid, + path + } + } + } catch (err) { + this.log.error('error parsing ipfs path - %e', err) + } + + this.log.error('invalid ipfs path %s', ipfsPath) + throw new InvalidValueError('Invalid value') + } + + async #findIpnsRecord (routingKey: Uint8Array, options: ResolveOptions = {}): Promise { + const records: Uint8Array[] = [] + const cached = await this.localStore.has(routingKey, options) + + if (cached) { + this.log('record is present in the cache') + + if (options.nocache !== true) { + try { + // check the local cache first + const { record, created } = await this.localStore.get(routingKey, options) + + this.log('record retrieved from cache') + + // validate the record + await ipnsValidator(routingKey, record) + + this.log('record was valid') + + // check the TTL + const ipnsRecord = unmarshalIPNSRecord(record) + + // IPNS TTL is in nanoseconds, convert to milliseconds, default to one + // hour + const ttlMs = Number((ipnsRecord.ttl ?? DEFAULT_TTL_NS) / 1_000_000n) + const ttlExpires = created.getTime() + ttlMs + + if (ttlExpires > Date.now()) { + // the TTL has not yet expired, return the cached record + this.log('record TTL was valid') + return ipnsRecord + } + + if (options.offline === true) { + // the TTL has expired but we are skipping the routing search + this.log('record TTL has been reached but we are resolving offline-only, returning record') + return ipnsRecord + } + + this.log('record TTL has been reached, searching routing for updates') + + // add the local record to our list of resolved record, and also + // search the routing for updates - the most up to date record will be + // returned + records.push(record) + } catch (err) { + this.log('cached record was invalid - %e', err) + await this.localStore.delete(routingKey, options) + } + } else { + this.log('ignoring local cache due to nocache=true option') + } + } + + if (options.offline === true) { + throw new NotFoundError('Record was not present in the cache or has expired') + } + + this.log('did not have record locally') + + let foundInvalid = 0 + + await Promise.all( + this.routers.map(async (router) => { + let record: Uint8Array + + try { + record = await router.get(routingKey, { + ...options, + validate: false + }) + } catch (err: any) { + this.log.error('error finding IPNS record - %e', err) + + return + } + + try { + await ipnsValidator(routingKey, record) + + records.push(record) + } catch (err) { + // we found a record, but the validator rejected it + foundInvalid++ + this.log.error('error finding IPNS record - %e', err) + } + }) + ) + + if (records.length === 0) { + if (foundInvalid > 0) { + throw new RecordsFailedValidationError(`${foundInvalid > 1 ? `${foundInvalid} records` : 'Record'} found for routing key ${foundInvalid > 1 ? 'were' : 'was'} invalid`) + } + + throw new NotFoundError('Could not find record for routing key') + } + + const record = records[ipnsSelector(routingKey, records)] + + await this.localStore.put(routingKey, record, options) + + return unmarshalIPNSRecord(record) + } +} diff --git a/packages/ipns/src/local-store.ts b/packages/ipns/src/local-store.ts new file mode 100644 index 000000000..4d325a8aa --- /dev/null +++ b/packages/ipns/src/local-store.ts @@ -0,0 +1,162 @@ +import { Record } from '@libp2p/kad-dht' +import { CustomProgressEvent } from 'progress-events' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { IPNSPublishMetadata } from './pb/metadata.js' +import { dhtRoutingKey, DHT_RECORD_PREFIX, ipnsMetadataKey } from './utils.js' +import type { DatastoreProgressEvents, GetOptions, PutOptions } from './routing/index.js' +import type { AbortOptions, Logger } from '@libp2p/interface' +import type { Datastore } from 'interface-datastore' + +export interface GetResult { + record: Uint8Array + created: Date +} + +export interface ListResult { + routingKey: Uint8Array + record: Uint8Array + created: Date + metadata?: IPNSPublishMetadata +} + +export interface ListOptions extends AbortOptions { + onProgress?(evt: DatastoreProgressEvents): void +} + +export interface LocalStore { + /** + * Put an IPNS record into the datastore + * + * @param routingKey - The routing key for the IPNS record + * @param marshaledRecord - The marshaled IPNS record + * @param options - options for the put operation including metadata + */ + put(routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions): Promise + get(routingKey: Uint8Array, options?: GetOptions): Promise + has(routingKey: Uint8Array, options?: AbortOptions): Promise + delete(routingKey: Uint8Array, options?: AbortOptions): Promise + /** + * List all IPNS records in the datastore + */ + list(options?: ListOptions): AsyncIterable +} + +/** + * Read/write IPNS records to the datastore as DHT records. + * + * This lets us publish IPNS records offline then serve them to the network + * later in response to DHT queries. + */ +export function localStore (datastore: Datastore, log: Logger): LocalStore { + return { + async put (routingKey: Uint8Array, marshalledRecord: Uint8Array, options: PutOptions = {}) { + try { + const key = dhtRoutingKey(routingKey) + + // don't overwrite existing, identical records as this will affect the + // TTL + try { + const existingBuf = await datastore.get(key) + const existingRecord = Record.deserialize(existingBuf) + + if (uint8ArrayEquals(existingRecord.value, marshalledRecord)) { + return + } + } catch (err: any) { + if (err.name !== 'NotFoundError') { + throw err + } + } + + // Marshal to libp2p record as the DHT does + const record = new Record(routingKey, marshalledRecord, new Date()) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) + const batch = datastore.batch() + batch.put(key, record.serialize()) + + if (options.metadata != null) { + // derive the datastore key for the IPNS metadata from the same routing key + batch.put(ipnsMetadataKey(routingKey), IPNSPublishMetadata.encode(options.metadata)) + } + await batch.commit(options) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + const key = dhtRoutingKey(routingKey) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:get')) + const buf = await datastore.get(key, options) + + // Unmarshal libp2p record as the DHT does + const record = Record.deserialize(buf) + + return { + record: record.value, + created: record.timeReceived + } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const key = dhtRoutingKey(routingKey) + return datastore.has(key, options) + }, + async delete (routingKey, options): Promise { + const key = dhtRoutingKey(routingKey) + const batch = datastore.batch() + batch.delete(key) + batch.delete(ipnsMetadataKey(routingKey)) + await batch.commit(options) + }, + async * list (options: ListOptions = {}): AsyncIterable { + try { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:list')) + + // Query all records with the DHT_RECORD_PREFIX + for await (const { key, value } of datastore.query({ + prefix: DHT_RECORD_PREFIX + }, options)) { + try { + // Deserialize the record + const libp2pRecord = Record.deserialize(value) + + // Extract the routing key from the datastore key + const keyString = key.toString() + const routingKeyBase32 = keyString.substring(DHT_RECORD_PREFIX.length) + const routingKey = uint8ArrayFromString(routingKeyBase32, 'base32') + + const metadataKey = ipnsMetadataKey(routingKey) + let metadata: IPNSPublishMetadata | undefined + try { + const metadataBuf = await datastore.get(metadataKey, options) + metadata = IPNSPublishMetadata.decode(metadataBuf) + } catch (err: any) { + log.error('Error deserializing metadata for %s - %e', routingKeyBase32, err) + } + + yield { + routingKey, + metadata, + record: libp2pRecord.value, + created: libp2pRecord.timeReceived + } + } catch (err) { + // Skip invalid records + log.error('Error deserializing record - %e', err) + } + } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + } + } +} diff --git a/packages/ipns/src/pb/metadata.proto b/packages/ipns/src/pb/metadata.proto new file mode 100644 index 000000000..143d9445b --- /dev/null +++ b/packages/ipns/src/pb/metadata.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +message IPNSPublishMetadata { + // The name of the key that was used to publish the record + string keyName = 1; + + // Lifetime is the duration of the signature validity in milliseconds + uint32 lifetime = 2; +} diff --git a/packages/ipns/src/pb/metadata.ts b/packages/ipns/src/pb/metadata.ts new file mode 100644 index 000000000..c4d25d59d --- /dev/null +++ b/packages/ipns/src/pb/metadata.ts @@ -0,0 +1,74 @@ +import { decodeMessage, encodeMessage, message } from 'protons-runtime' +import type { Codec, DecodeOptions } from 'protons-runtime' +import type { Uint8ArrayList } from 'uint8arraylist' + +export interface IPNSPublishMetadata { + keyName: string + lifetime: number +} + +export namespace IPNSPublishMetadata { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.keyName != null && obj.keyName !== '')) { + w.uint32(10) + w.string(obj.keyName) + } + + if ((obj.lifetime != null && obj.lifetime !== 0)) { + w.uint32(16) + w.uint32(obj.lifetime) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length, opts = {}) => { + const obj: any = { + keyName: '', + lifetime: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.keyName = reader.string() + break + } + case 2: { + obj.lifetime = reader.uint32() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, IPNSPublishMetadata.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): IPNSPublishMetadata => { + return decodeMessage(buf, IPNSPublishMetadata.codec(), opts) + } +} diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index 0513def50..87bf60393 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -1,11 +1,12 @@ import type { HeliaRoutingProgressEvents } from './helia.js' -import type { DatastoreProgressEvents } from './local-store.js' +import type { DatastoreProgressEvents } from '../index.js' import type { PubSubProgressEvents } from './pubsub.js' +import type { IPNSPublishMetadata } from '../pb/metadata.ts' import type { AbortOptions } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' export interface PutOptions extends AbortOptions, ProgressOptions { - + metadata?: IPNSPublishMetadata } export interface GetOptions extends AbortOptions, ProgressOptions { diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index 6136a4af2..6a7f4c049 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -1,96 +1,18 @@ -import { Record } from '@libp2p/kad-dht' -import { Key } from 'interface-datastore' -import { CustomProgressEvent } from 'progress-events' -import { equals as uint8ArrayEquals } from 'uint8arrays/equals' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { GetOptions, PutOptions } from '../routing/index.js' -import type { AbortOptions } from '@libp2p/interface' -import type { Datastore } from 'interface-datastore' -import type { ProgressEvent } from 'progress-events' - -function dhtRoutingKey (key: Uint8Array): Key { - return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) -} - -export type DatastoreProgressEvents = - ProgressEvent<'ipns:routing:datastore:put'> | - ProgressEvent<'ipns:routing:datastore:get'> | - ProgressEvent<'ipns:routing:datastore:error', Error> - -export interface GetResult { - record: Uint8Array - created: Date -} - -export interface LocalStore { - put(routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions): Promise - get(routingKey: Uint8Array, options?: GetOptions): Promise - has(routingKey: Uint8Array, options?: AbortOptions): Promise - delete(routingKey: Uint8Array, options?: AbortOptions): Promise -} +import type { LocalStore } from '../local-store.ts' +import type { IPNSRouting, PutOptions, GetOptions } from './index.ts' /** - * Returns an IPNSRouting implementation that reads/writes IPNS records to the - * datastore as DHT records. This lets us publish IPNS records offline then - * serve them to the network later in response to DHT queries. + * Returns an IPNSRouting implementation that reads/writes to the local store */ -export function localStore (datastore: Datastore): LocalStore { +export function localStoreRouting (localStore: LocalStore): IPNSRouting { return { - async put (routingKey: Uint8Array, marshalledRecord: Uint8Array, options: PutOptions = {}) { - try { - const key = dhtRoutingKey(routingKey) - - // don't overwrite existing, identical records as this will affect the - // TTL - try { - const existingBuf = await datastore.get(key) - const existingRecord = Record.deserialize(existingBuf) - - if (uint8ArrayEquals(existingRecord.value, marshalledRecord)) { - return - } - } catch (err: any) { - if (err.name !== 'NotFoundError') { - throw err - } - } - - // Marshal to libp2p record as the DHT does - const record = new Record(routingKey, marshalledRecord, new Date()) - - options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) - await datastore.put(key, record.serialize(), options) - } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) - throw err - } + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions): Promise { + await localStore.put(routingKey, marshaledRecord, options) }, - async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { - try { - const key = dhtRoutingKey(routingKey) - - options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:get')) - const buf = await datastore.get(key, options) + async get (routingKey: Uint8Array, options?: GetOptions): Promise { + const { record } = await localStore.get(routingKey, options) - // Unmarshal libp2p record as the DHT does - const record = Record.deserialize(buf) - - return { - record: record.value, - created: record.timeReceived - } - } catch (err: any) { - options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) - throw err - } - }, - async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { - const key = dhtRoutingKey(routingKey) - return datastore.has(key, options) - }, - async delete (routingKey, options): Promise { - const key = dhtRoutingKey(routingKey) - return datastore.delete(key, options) + return record } } } diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index 33b37a1ec..81e277800 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -8,10 +8,10 @@ import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { InvalidTopicError } from '../errors.js' -import { localStore } from './local-store.js' +import { localStore } from '../local-store.js' import type { GetOptions, IPNSRouting, PutOptions } from './index.js' -import type { LocalStore } from './local-store.js' -import type { PeerId, PublicKey, TypedEventTarget } from '@libp2p/interface' +import type { LocalStore } from '../local-store.js' +import type { PeerId, PublicKey, TypedEventTarget, ComponentLogger } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { MultihashDigest } from 'multiformats/hashes/interface' import type { ProgressEvent } from 'progress-events' @@ -43,6 +43,7 @@ export interface PubSub extends TypedEventTarget { export interface PubsubRoutingComponents { datastore: Datastore + logger: ComponentLogger libp2p: { peerId: PeerId services: { @@ -64,7 +65,7 @@ class PubSubRouting implements IPNSRouting { constructor (components: PubsubRoutingComponents) { this.subscriptions = [] - this.localStore = localStore(components.datastore) + this.localStore = localStore(components.datastore, components.logger.forComponent('helia:ipns:local-store')) this.peerId = components.libp2p.peerId this.pubsub = components.libp2p.services.pubsub diff --git a/packages/ipns/src/utils.ts b/packages/ipns/src/utils.ts index 182863dc7..9180a0dcf 100644 --- a/packages/ipns/src/utils.ts +++ b/packages/ipns/src/utils.ts @@ -1,3 +1,7 @@ +import { Key } from 'interface-datastore' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { DHT_EXPIRY_MS, REPUBLISH_THRESHOLD } from './constants.ts' +import type { IPNSRecord } from 'ipns' import type { MultihashDigest } from 'multiformats/hashes/interface' export const IDENTITY_CODEC = 0x0 @@ -8,3 +12,46 @@ export const IPNS_STRING_PREFIX = '/ipns/' export function isCodec (digest: MultihashDigest, codec: T): digest is MultihashDigest { return digest.code === codec } + +export const DHT_RECORD_PREFIX = '/dht/record/' +export const IPNS_METADATA_PREFIX = '/ipns/metadata/' + +export function dhtRoutingKey (key: Uint8Array): Key { + return new Key(DHT_RECORD_PREFIX + uint8ArrayToString(key, 'base32'), false) +} + +/** + * Calculate the datastore key for IPNS record metadata + * + * @param key - The DHT routing key for the IPNS record as defined in + * https://specs.ipfs.tech/ipns/ipns-record/#routing-record + * + * @example + * + * ```ts + * const key = multihashToIPNSRoutingKey(privKey.publicKey.toMultihash()) + * const metadataKey = ipnsMetadataKey(key) + * ``` + * @returns The local storage key for IPNS record metadata + */ +export function ipnsMetadataKey (key: Uint8Array): Key { + return new Key(IPNS_METADATA_PREFIX + uint8ArrayToString(key, 'base32'), false) +} + +export function shouldRepublish (ipnsRecord: IPNSRecord, created: Date): boolean { + const now = Date.now() + const dhtExpiry = created.getTime() + DHT_EXPIRY_MS + const recordExpiry = new Date(ipnsRecord.validity).getTime() + + // If the DHT expiry is within the threshold, republish it + if (dhtExpiry - now < REPUBLISH_THRESHOLD) { + return true + } + + // If the record expiry (based on validity/lifetime) is within the threshold, republish it + if (recordExpiry - now < REPUBLISH_THRESHOLD) { + return true + } + + return false +} diff --git a/packages/ipns/test/fixtures/create-ipns.ts b/packages/ipns/test/fixtures/create-ipns.ts new file mode 100644 index 000000000..c72c2979d --- /dev/null +++ b/packages/ipns/test/fixtures/create-ipns.ts @@ -0,0 +1,68 @@ +import { TypedEventEmitter } from '@libp2p/interface' +import { keychain } from '@libp2p/keychain' +import { defaultLogger } from '@libp2p/logger' +import { MemoryDatastore } from 'datastore-core' +import { stubInterface } from 'sinon-ts' +import { IPNS } from '../../src/ipns.ts' +import type { IPNSRouting } from '../../src/index.js' +import type { HeliaEvents, Routing } from '@helia/interface' +import type { Keychain, KeychainInit } from '@libp2p/keychain' +import type { Logger } from '@libp2p/logger' +import type { Datastore } from 'interface-datastore' +import type { StubbedInstance } from 'sinon-ts' + +export interface CreateIPNSResult { + name: IPNS + customRouting: StubbedInstance + heliaRouting: StubbedInstance + ipnsKeychain: Keychain + datastore: Datastore, + log: Logger + events: TypedEventEmitter +} + +export async function createIPNS (): Promise { + const datastore = new MemoryDatastore() + + // Create stubbed instances if not provided + const customRouting = stubInterface() + customRouting.get.throws(new Error('Not found')) + + const heliaRouting = stubInterface() + + const logger = defaultLogger() + const keychainInit: KeychainInit = { + pass: 'very-strong-password' + } + const ipnsKeychain = keychain(keychainInit)({ + datastore, + logger + }) + + const events = new TypedEventEmitter() + + const name = new IPNS({ + datastore, + routing: heliaRouting, + libp2p: { + status: 'stopped', + services: { + keychain: ipnsKeychain + } + } as any, + logger, + events + }, { + routers: [customRouting] + }) + + return { + name, + customRouting, + heliaRouting, + ipnsKeychain, + datastore, + log: logger.forComponent('helia:ipns:test'), + events + } +} diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 49b9f245f..402a1a6d6 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -1,72 +1,57 @@ /* eslint-env mocha */ -import { generateKeyPair } from '@libp2p/crypto/keys' -import { defaultLogger } from '@libp2p/logger' +import { start, stop } from '@libp2p/interface' import { expect } from 'aegir/chai' -import { MemoryDatastore } from 'datastore-core' import { base36 } from 'multiformats/bases/base36' import { CID } from 'multiformats/cid' import Sinon from 'sinon' -import { stubInterface } from 'sinon-ts' -import { ipns } from '../src/index.js' -import type { IPNS, IPNSRouting } from '../src/index.js' -import type { Routing } from '@helia/interface' -import type { DNS } from '@multiformats/dns' -import type { StubbedInstance } from 'sinon-ts' +import { localStore } from '../src/local-store.js' +import { createIPNS } from './fixtures/create-ipns.js' +import type { CreateIPNSResult } from './fixtures/create-ipns.js' +import type { IPNS } from '../src/ipns.js' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') describe('publish', () => { let name: IPNS - let customRouting: StubbedInstance - let heliaRouting: StubbedInstance - let dns: StubbedInstance + let customRouting: any + let heliaRouting: any + let result: CreateIPNSResult beforeEach(async () => { - const datastore = new MemoryDatastore() - customRouting = stubInterface() - customRouting.get.throws(new Error('Not found')) - heliaRouting = stubInterface() - dns = stubInterface() - - name = ipns({ - datastore, - routing: heliaRouting, - dns, - logger: defaultLogger() - }, { - routers: [ - customRouting - ] - }) + result = await createIPNS() + name = result.name + customRouting = result.customRouting + heliaRouting = result.heliaRouting + + await start(name) + }) + + afterEach(async () => { + await stop(name) }) it('should publish an IPNS record with the default params', async function () { - const key = await generateKeyPair('Ed25519') - const ipnsEntry = await name.publish(key, cid) + const keyName = 'test-key-1' + const ipnsEntry = await name.publish(keyName, cid) - expect(ipnsEntry).to.have.property('sequence', 1n) - expect(ipnsEntry).to.have.property('ttl', 300_000_000_000n) // 5 minutes + expect(ipnsEntry.record).to.have.property('sequence', 1n) + expect(ipnsEntry.record).to.have.property('ttl', 300_000_000_000n) // 5 minutes }) it('should publish an IPNS record with a custom lifetime params', async function () { - const key = await generateKeyPair('Ed25519') + const keyName = 'test-key-2' const lifetime = 123000 - // lifetime is used to calculate the validity timestamp - const ipnsEntry = await name.publish(key, cid, { + const ipnsEntry = await name.publish(keyName, cid, { lifetime }) - expect(ipnsEntry).to.have.property('sequence', 1n) + expect(ipnsEntry.record).to.have.property('sequence', 1n) // Calculate expected validity as a Date object const expectedValidity = new Date(Date.now() + lifetime) - - const actualValidity = new Date(ipnsEntry.validity) - + const actualValidity = new Date(ipnsEntry.record.validity) const timeDifference = Math.abs(actualValidity.getTime() - expectedValidity.getTime()) - - // Allow a tolerance of 1 second (1000 milliseconds) expect(timeDifference).to.be.lessThan(1000) expect(heliaRouting.put.called).to.be.true() @@ -74,23 +59,23 @@ describe('publish', () => { }) it('should publish an IPNS record with a custom ttl params', async function () { - const key = await generateKeyPair('Ed25519') + const keyName = 'test-key-3' const ttl = 1000 // override the default ttl - const ipnsEntry = await name.publish(key, cid, { + const ipnsEntry = await name.publish(keyName, cid, { ttl }) - expect(ipnsEntry).to.have.property('sequence', 1n) - expect(ipnsEntry).to.have.property('ttl', BigInt(ttl * 1e+6)) + expect(ipnsEntry.record).to.have.property('sequence', 1n) + expect(ipnsEntry.record).to.have.property('ttl', BigInt(ttl * 1e+6)) expect(heliaRouting.put.called).to.be.true() expect(customRouting.put.called).to.be.true() }) it('should publish a record offline', async () => { - const key = await generateKeyPair('Ed25519') - await name.publish(key, cid, { + const keyName = 'test-key-4' + await name.publish(keyName, cid, { offline: true }) @@ -99,9 +84,9 @@ describe('publish', () => { }) it('should emit progress events', async function () { - const key = await generateKeyPair('Ed25519') + const keyName = 'test-key-5' const onProgress = Sinon.stub() - await name.publish(key, cid, { + await name.publish(keyName, cid, { onProgress }) @@ -109,21 +94,21 @@ describe('publish', () => { }) it('should publish recursively', async () => { - const key = await generateKeyPair('Ed25519') - const record = await name.publish(key, cid, { + const keyName1 = 'test-key-6' + const record = await name.publish(keyName1, cid, { offline: true }) - expect(record.value).to.equal(`/ipfs/${cid.toV1().toString()}`) + expect(record.record.value).to.equal(`/ipfs/${cid.toV1().toString()}`) - const recursiveKey = await generateKeyPair('Ed25519') - const recursiveRecord = await name.publish(recursiveKey, key.publicKey, { + const keyName2 = 'test-key-7' + const recursiveRecord = await name.publish(keyName2, record.publicKey, { offline: true }) - expect(recursiveRecord.value).to.equal(`/ipns/${key.publicKey.toCID().toString(base36)}`) + expect(recursiveRecord.record.value).to.equal(`/ipns/${record.publicKey.toCID().toString(base36)}`) - const recursiveResult = await name.resolve(recursiveKey.publicKey) + const recursiveResult = await name.resolve(record.publicKey) expect(recursiveResult.cid.toString()).to.equal(cid.toV1().toString()) }) @@ -131,16 +116,89 @@ describe('publish', () => { const path = '/foo/bar/baz' const fullPath = `/ipfs/${cid}/${path}` - const key = await generateKeyPair('Ed25519') - const record = await name.publish(key, fullPath, { + const keyName = 'test-key-8' + const record = await name.publish(keyName, fullPath, { offline: true }) - expect(record.value).to.equal(fullPath) + expect(record.record.value).to.equal(fullPath) - const result = await name.resolve(key.publicKey) + const result = await name.resolve(record.publicKey) expect(result.cid.toString()).to.equal(cid.toString()) expect(result.path).to.equal(path) }) + + describe('localStore error handling', () => { + it('should handle datastore errors during publish', async () => { + await start(name) + + // Stub localStore.get to throw an error + const store = localStore(result.datastore, result.log) + const getStub = Sinon.stub(store, 'get').rejects(new Error('Datastore get failed')) + const hasStub = Sinon.stub(store, 'has').resolves(true) + + // Override the localStore on the IPNS instance + // @ts-ignore + name.localStore = store + + const keyName = 'test-key-error' + await expect(name.publish(keyName, cid)).to.be.rejectedWith('Datastore get failed') + + expect(hasStub.called).to.be.true() + expect(getStub.called).to.be.true() + }) + + it('should handle datastore put errors during publish', async () => { + await start(name) + + // Stub datastore.put to throw an error + const putStub = Sinon.stub(result.datastore, 'put').rejects(new Error('Datastore put failed')) + const hasStub = Sinon.stub(result.datastore, 'has').resolves(false) + + const keyName = 'test-key-put-error' + await expect(name.publish(keyName, cid)).to.be.rejectedWith('Datastore put failed') + + expect(hasStub.called).to.be.true() + expect(putStub.called).to.be.true() + }) + + it('should emit error progress events when localStore fails', async () => { + await start(name) + + const progressEvents: any[] = [] + + const putStub = Sinon.stub(result.datastore, 'get').rejects(new Error('Storage error')) + const hasStub = Sinon.stub(result.datastore, 'has').resolves(false) + + const keyName = 'test-key-progress-error' + + await expect(name.publish(keyName, cid, { + onProgress: (evt) => progressEvents.push(evt) + })).to.be.rejectedWith('Storage error') + + expect(hasStub.called).to.be.true() + expect(putStub.called).to.be.true() + + // Check if error progress event was emitted by localStore + const errorEvent = progressEvents.find(evt => evt.type === 'ipns:routing:datastore:error') + expect(errorEvent).to.exist() + expect(errorEvent.detail.message).to.equal('Storage error') + }) + + it('should handle network timeouts in localStore', async () => { + await start(name) + + // Create a timeout error + const timeoutError = new Error('Network timeout') + timeoutError.name = 'TimeoutError' + + const hasStub = Sinon.stub(result.datastore, 'has').rejects(timeoutError) + + const keyName = 'test-key-timeout' + await expect(name.publish(keyName, cid)).to.be.rejectedWith('Network timeout') + + expect(hasStub.called).to.be.true() + }) + }) }) diff --git a/packages/ipns/test/republish.spec.ts b/packages/ipns/test/republish.spec.ts index 2103e027f..4c9910e1e 100644 --- a/packages/ipns/test/republish.spec.ts +++ b/packages/ipns/test/republish.spec.ts @@ -1,107 +1,439 @@ /* eslint-env mocha */ import { generateKeyPair } from '@libp2p/crypto/keys' -import { defaultLogger } from '@libp2p/logger' +import { start, stop } from '@libp2p/interface' import { expect } from 'aegir/chai' -import { MemoryDatastore } from 'datastore-core' -import { createIPNSRecord } from 'ipns' -import { base32 } from 'multiformats/bases/base32' -import { base36 } from 'multiformats/bases/base36' +import { createIPNSRecord, marshalIPNSRecord, unmarshalIPNSRecord, multihashToIPNSRoutingKey } from 'ipns' import { CID } from 'multiformats/cid' -import { stubInterface } from 'sinon-ts' -import { ipns } from '../src/index.js' -import { IPNS_STRING_PREFIX } from '../src/utils.js' -import type { IPNS, IPNSRouting } from '../src/index.js' -import type { Routing } from '@helia/interface' -import type { DNS } from '@multiformats/dns' -import type { StubbedInstance } from 'sinon-ts' - -describe('republishRecord', () => { +import sinon from 'sinon' +import { localStore } from '../src/local-store.js' +import { createIPNS } from './fixtures/create-ipns.js' +import type { IPNS } from '../src/ipns.js' +import type { CreateIPNSResult } from './fixtures/create-ipns.js' + +// Helper to await until a stub is called +function waitForStubCall (stub: sinon.SinonStub, callCount = 1): Promise { + return new Promise((resolve) => { + const check = (): void => { + if (stub.callCount >= callCount) { + resolve() + } else { + setTimeout(check, 1) + } + } + check() + }) +} + +describe('republish', () => { const testCid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') let name: IPNS - let customRouting: StubbedInstance - let heliaRouting: StubbedInstance - let dns: StubbedInstance + let result: CreateIPNSResult + let putStubCustom: sinon.SinonStub + let putStubHelia: sinon.SinonStub beforeEach(async () => { - const datastore = new MemoryDatastore() - customRouting = stubInterface() - customRouting.get.throws(new Error('Not found')) - heliaRouting = stubInterface() - dns = stubInterface() - - name = ipns( - { - datastore, - routing: heliaRouting, - dns, - logger: defaultLogger() - }, - { - routers: [customRouting] - } - ) - }) + result = await createIPNS() + name = result.name - it('should throw an error when attempting to republish with an invalid key', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const otherEd25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - await expect(name.republishRecord(otherEd25519Key.publicKey.toMultihash(), ed25519Record)).to.be.rejected + // Stub the routers by default + putStubCustom = sinon.stub().resolves() + putStubHelia = sinon.stub().resolves() + // @ts-ignore + result.customRouting.put = putStubCustom + // @ts-ignore + result.heliaRouting.put = putStubHelia }) - it('should republish using the embedded public key', async () => { - const rsaKey = await generateKeyPair('RSA') // RSA will embed the public key in the record - const otherKey = await generateKeyPair('RSA') - const rsaRecord = await createIPNSRecord(rsaKey, testCid, 1n, 24 * 60 * 60 * 1000) - await expect(name.republishRecord(otherKey.publicKey.toMultihash(), rsaRecord)).to.not.be.rejected + afterEach(async () => { + await stop(name) + sinon.restore() + sinon.reset() }) - it('should republish a record using provided public key', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - await expect(name.republishRecord(ed25519Key.publicKey.toMultihash(), ed25519Record)).to.not.be.rejected - }) + describe('basic functionality', () => { + it('should start republishing when called', async () => { + // Create a test record and store it in the real datastore + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) - it('should republish a record using a string key (base58btc encoded multihash)', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - const keyString = ed25519Key.publicKey.toString() - await expect(name.republishRecord(keyString, ed25519Record)).to.not.be.rejected - }) + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore using the localStore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + // Start republishing + await start(name) + await waitForStubCall(putStubCustom) + + // Only check custom router for most tests + expect(putStubCustom.called).to.be.true() + }) + + it('should call all routers for republish', async () => { + // Create a test record and store it in the real datastore + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore using the localStore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + // Start republishing + await start(name) + + await Promise.all([ + waitForStubCall(putStubCustom), + waitForStubCall(putStubHelia) + ]) + + // Check both routers + expect(putStubCustom.called).to.be.true() + expect(putStubHelia.called).to.be.true() + expect(putStubCustom.firstCall.args[0]).to.deep.equal(routingKey) + expect(putStubHelia.firstCall.args[0]).to.deep.equal(routingKey) + }) + + it('should republish records with valid metadata', async () => { + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) - it('should republish a record using a string key (base36 encoded CID)', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - const keyString = ed25519Key.publicKey.toCID().toString(base36) - await expect(name.republishRecord(keyString, ed25519Record)).to.not.be.rejected + // Start publishing + await start(name) + await waitForStubCall(putStubCustom) + + // Verify the record was republished with incremented sequence + expect(putStubCustom.called).to.be.true() + const callArgs = putStubCustom.firstCall.args + expect(callArgs[0]).to.deep.equal(routingKey) + + const republishedRecord = unmarshalIPNSRecord(callArgs[1]) + expect(republishedRecord.sequence).to.equal(2n) // Incremented from 1n + }) }) - it('should republish a record using a string key (base32 encoded CID)', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - const keyString = ed25519Key.publicKey.toCID().toString(base32) - await expect(name.republishRecord(keyString, ed25519Record)).to.not.be.rejected + describe('record processing', () => { + it('should skip records without metadata', async () => { + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Store the record without metadata (simulate old records) + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record)) // No metadata + + await start(name) + await new Promise(resolve => setTimeout(resolve, 20)) + + // Verify no records were republished + expect(putStubCustom.called).to.be.false() + }) + + it('should handle invalid records gracefully', async () => { + const routingKey = new Uint8Array([1, 2, 3, 4]) + + // Store an invalid record in the datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, new Uint8Array([255, 255, 255]), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + await start(name) + await new Promise(resolve => setTimeout(resolve, 20)) + + // Verify no records were republished due to error + expect(putStubCustom.called).to.be.false() + }) + + it('should increment sequence numbers correctly', async () => { + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 5n, 24 * 60 * 60 * 1000) // Start with sequence 5 + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + await start(name) + await waitForStubCall(putStubCustom) + + expect(putStubCustom.called).to.be.true() + + const callArgs = putStubCustom.firstCall.args + const republishedRecord = unmarshalIPNSRecord(callArgs[1]) + expect(republishedRecord.sequence).to.equal(6n) // Incremented from 5n + }) }) - it('should republish a record using a string key (base36 encoded CID) prefixed with /ipns/', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) - const keyString = `${IPNS_STRING_PREFIX}${ed25519Key.publicKey.toCID().toString(base36)}` - await expect(name.republishRecord(keyString, ed25519Record)).to.not.be.rejected + describe('TTL and lifetime', () => { + it('should use existing TTL from records', async () => { + const key = await generateKeyPair('Ed25519') + const customTtl = BigInt(10 * 60 * 1000) * 1_000_000n // 10 minutes in nanoseconds + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000, { ttlNs: customTtl }) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + await start(name) + await waitForStubCall(putStubCustom) + + // Verify the record was republished with incremented sequence + expect(putStubCustom.called).to.be.true() + const callArgs = putStubCustom.firstCall.args + expect(callArgs[0]).to.deep.equal(routingKey) + + const republishedRecord = unmarshalIPNSRecord(callArgs[1]) + expect(republishedRecord.sequence).to.equal(2n) // Incremented from 1n + expect(republishedRecord.ttl).to.equal(customTtl) + }) + + it('should use default TTL when not present', async () => { + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + await start(name) + await waitForStubCall(putStubCustom) + + expect(putStubCustom.called).to.be.true() + const callArgs = putStubCustom.firstCall.args + const republishedRecord = unmarshalIPNSRecord(callArgs[1]) + expect(republishedRecord.ttl).to.equal(5n * 60n * 1000n * 1_000_000n) // Default TTL + }) + + it('should use metadata lifetime', async () => { + const key = await generateKeyPair('Ed25519') + const customLifetime = 5 * 1000 // 5 seconds + const record = await createIPNSRecord(key, testCid, 1n, customLifetime) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key into the real keychain + await result.ipnsKeychain.importKey('test-key', key) + + // Store the record in the real datastore + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'test-key', + lifetime: customLifetime + } + }) + + await start(name) + await waitForStubCall(putStubCustom) + + const expectedValidity = Date.now() + customLifetime + + expect(putStubCustom.called).to.be.true() + + const callArgs = putStubCustom.firstCall.args + const republishedRecord = unmarshalIPNSRecord(callArgs[1]) + + // Check that the validity is set to the custom lifetime + const actualValidity = new Date(republishedRecord.validity) + + const timeDiff = Math.abs(actualValidity.getTime() - expectedValidity) + expect(timeDiff).to.be.lessThan(200) + }) }) - it('should emit progress events on error', async () => { - const ed25519Key = await generateKeyPair('Ed25519') - const otherEd25519Key = await generateKeyPair('Ed25519') - const ed25519Record = await createIPNSRecord(ed25519Key, testCid, 1n, 24 * 60 * 60 * 1000) + describe('error handling', () => { + it('should skip republishing records with missing key', async () => { + const key = await generateKeyPair('Ed25519') + const record = await createIPNSRecord(key, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Store the record in the real datastore (but don't import the key) + const store = localStore(result.datastore, result.log) + await store.put(routingKey, marshalIPNSRecord(record), { + metadata: { + keyName: 'missing-key', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + const interval = 5 + await start(name) + await new Promise(resolve => setTimeout(resolve, interval + 10)) + // Should not republish due to keychain error (key not found) + expect(putStubCustom.called).to.be.false() + expect(putStubHelia.called).to.be.false() + }) + + it('should handle localStore.list() errors during republish', async () => { + // Stub localStore to throw error during list operation + const store = localStore(result.datastore, result.log) + const listStub = sinon.stub(store, 'list').throws(new Error('Datastore list failed')) + + // Override the localStore on the IPNS instance + // @ts-ignore + name.localStore = store - await expect( - name.republishRecord(otherEd25519Key.publicKey.toMultihash(), ed25519Record, { - onProgress: (evt) => { - expect(evt.type).to.equal('ipns:republish:error') + await start(name) + await new Promise(resolve => setTimeout(resolve, 20)) + + expect(listStub.called).to.be.true() + // Should not republish due to list error + expect(putStubCustom.called).to.be.false() + expect(putStubHelia.called).to.be.false() + + // Check if error progress event was emitted + // const errorEvent = progressEvents.find(evt => evt.type === 'ipns:republish:error') + // expect(errorEvent).to.exist() + }) + + it.skip('should emit error progress events when localStore.list() fails', async () => { + const store = localStore(result.datastore, result.log) + const progressEvents: any[] = [] + + // Stub list to emit error progress event and then throw + // eslint-disable-next-line require-yield + const listStub = sinon.stub(store, 'list').callsFake(async function * (options) { + // Simulate the error progress event emission + options?.onProgress?.({ + type: 'ipns:routing:datastore:error', + detail: new Error('List operation failed') + }) + throw new Error('List operation failed') + }) + + // Override the localStore + // @ts-ignore + name.localStore = store + + await start(name) + await new Promise(resolve => setTimeout(resolve, 20)) + + expect(listStub.called).to.be.true() + expect(putStubCustom.called).to.be.false() + expect(putStubHelia.called).to.be.false() + + // Check if datastore error progress event was emitted + const datastoreErrorEvent = progressEvents.find(evt => evt.type === 'ipns:routing:datastore:error') + expect(datastoreErrorEvent).to.exist() + expect(datastoreErrorEvent.detail.message).to.equal('List operation failed') + }) + + it('should handle corrupt record data during republish iteration', async () => { + const key = await generateKeyPair('Ed25519') + const routingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) + + // Import the key + await result.ipnsKeychain.importKey('test-key', key) + + const store = localStore(result.datastore, result.log) + + // Store corrupt record data that will fail to unmarshal + await store.put(routingKey, new Uint8Array([255, 255, 255]), { + metadata: { + keyName: 'test-key', + lifetime: 24 * 60 * 60 * 1000 } }) - ).to.eventually.be.rejected.with.property('name', 'SignatureVerificationError') + + await start(name) + await new Promise(resolve => setTimeout(resolve, 20)) + + // Should not republish due to unmarshal error + expect(putStubCustom.called).to.be.false() + expect(putStubHelia.called).to.be.false() + }) + + it('should continue republishing other records when one record fails', async () => { + const key1 = await generateKeyPair('Ed25519') + const key2 = await generateKeyPair('Ed25519') + const record2 = await createIPNSRecord(key2, testCid, 1n, 24 * 60 * 60 * 1000) + const routingKey1 = multihashToIPNSRoutingKey(key1.publicKey.toMultihash()) + const routingKey2 = multihashToIPNSRoutingKey(key2.publicKey.toMultihash()) + + // Import both keys + await result.ipnsKeychain.importKey('test-key-1', key1) + await result.ipnsKeychain.importKey('test-key-2', key2) + + const store = localStore(result.datastore, result.log) + + // Store one valid record and one corrupt record + await store.put(routingKey1, new Uint8Array([255, 255, 255]), { + metadata: { + keyName: 'test-key-1', + lifetime: 24 * 60 * 60 * 1000 + } + }) + await store.put(routingKey2, marshalIPNSRecord(record2), { + metadata: { + keyName: 'test-key-2', + lifetime: 24 * 60 * 60 * 1000 + } + }) + + await start(name) + await waitForStubCall(putStubCustom) + + // Should republish the valid record despite the corrupt one + expect(putStubCustom.called).to.be.true() + expect(putStubHelia.called).to.be.true() + }) }) }) diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index a0dab0fee..1e9603846 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -1,21 +1,18 @@ /* eslint-env mocha */ import { generateKeyPair } from '@libp2p/crypto/keys' +import { start, stop } from '@libp2p/interface' import { Record } from '@libp2p/kad-dht' -import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' -import { MemoryDatastore } from 'datastore-core' import { Key } from 'interface-datastore' import { createIPNSRecord, createIPNSRecordWithExpiration, marshalIPNSRecord, multihashToIPNSRoutingKey, unmarshalIPNSRecord } from 'ipns' import drain from 'it-drain' import { CID } from 'multiformats/cid' import Sinon from 'sinon' -import { stubInterface } from 'sinon-ts' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { ipns } from '../src/index.js' -import type { IPNS, IPNSRouting } from '../src/index.js' +import { createIPNS } from './fixtures/create-ipns.js' +import type { IPNS } from '../src/index.js' import type { Routing } from '@helia/interface' -import type { DNS } from '@multiformats/dns' import type { Datastore } from 'interface-datastore' import type { StubbedInstance } from 'sinon-ts' @@ -23,40 +20,34 @@ const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') describe('resolve', () => { let name: IPNS - let customRouting: StubbedInstance + let customRouting: any let datastore: Datastore let heliaRouting: StubbedInstance - let dns: StubbedInstance beforeEach(async () => { - datastore = new MemoryDatastore() - customRouting = stubInterface() - customRouting.get.throws(new Error('Not found')) - heliaRouting = stubInterface() - dns = stubInterface() - - name = ipns({ - datastore, - routing: heliaRouting, - dns, - logger: defaultLogger() - }, { - routers: [ - customRouting - ] - }) + const result = await createIPNS() + name = result.name + customRouting = result.customRouting + heliaRouting = result.heliaRouting + datastore = result.datastore + + await start(name) + }) + + afterEach(async () => { + await stop(name) }) it('should resolve a record', async () => { - const key = await generateKeyPair('Ed25519') - const record = await name.publish(key, cid) + const keyName = 'test-key' + const { record, publicKey } = await name.publish(keyName, cid) // empty the datastore to ensure we resolve using the routing await drain(datastore.deleteMany(datastore.queryKeys({}))) heliaRouting.get.resolves(marshalIPNSRecord(record)) - const resolvedValue = await name.resolve(key.publicKey) + const resolvedValue = await name.resolve(publicKey) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) expect(heliaRouting.get.called).to.be.true() @@ -64,13 +55,13 @@ describe('resolve', () => { }) it('should resolve a record offline', async () => { - const key = await generateKeyPair('Ed25519') - await name.publish(key, cid) + const keyName = 'test-key' + const { publicKey } = await name.publish(keyName, cid) expect(heliaRouting.put.called).to.be.true() expect(customRouting.put.called).to.be.true() - const resolvedValue = await name.resolve(key.publicKey, { + const resolvedValue = await name.resolve(publicKey, { offline: true }) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) @@ -83,12 +74,12 @@ describe('resolve', () => { const cachePutSpy = Sinon.spy(datastore, 'put') const cacheGetSpy = Sinon.spy(datastore, 'get') - const key = await generateKeyPair('Ed25519') - const record = await name.publish(key, cid) + const keyName = 'test-key' + const { record, publicKey } = await name.publish(keyName, cid) heliaRouting.get.resolves(marshalIPNSRecord(record)) - const resolvedValue = await name.resolve(key.publicKey, { + const resolvedValue = await name.resolve(publicKey, { nocache: true }) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) @@ -103,10 +94,10 @@ describe('resolve', () => { it('should retrieve from local cache when resolving a record', async () => { const cacheGetSpy = Sinon.spy(datastore, 'get') - const key = await generateKeyPair('Ed25519') - await name.publish(key, cid) + const keyName = 'test-key' + const { publicKey } = await name.publish(keyName, cid) - const resolvedValue = await name.resolve(key.publicKey) + const resolvedValue = await name.resolve(publicKey) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) expect(heliaRouting.get.called).to.be.false() @@ -115,31 +106,33 @@ describe('resolve', () => { }) it('should resolve a recursive record', async () => { - const key1 = await generateKeyPair('Ed25519') - const key2 = await generateKeyPair('Ed25519') - await name.publish(key2, cid) - await name.publish(key1, key2.publicKey) + const keyName1 = 'key1' + const keyName2 = 'key2' - const resolvedValue = await name.resolve(key1.publicKey) + const { publicKey: publicKey2 } = await name.publish(keyName2, cid) + const { publicKey: publicKey1 } = await name.publish(keyName1, publicKey2) + + const resolvedValue = await name.resolve(publicKey1) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) }) it('should resolve a recursive record with path', async () => { - const key1 = await generateKeyPair('Ed25519') - const key2 = await generateKeyPair('Ed25519') - await name.publish(key2, cid) - await name.publish(key1, key2.publicKey) + const keyName1 = 'key1' + const keyName2 = 'key2' + + const { publicKey: publicKey2 } = await name.publish(keyName2, cid) + const { publicKey: publicKey1 } = await name.publish(keyName1, publicKey2) - const resolvedValue = await name.resolve(key1.publicKey) + const resolvedValue = await name.resolve(publicKey1) expect(resolvedValue.cid.toString()).to.equal(cid.toV1().toString()) }) it('should emit progress events', async function () { const onProgress = Sinon.stub() - const key = await generateKeyPair('Ed25519') - await name.publish(key, cid) + const keyName = 'test-key' + const { publicKey } = await name.publish(keyName, cid) - await name.resolve(key.publicKey, { + await name.resolve(publicKey, { onProgress }) @@ -190,18 +183,13 @@ describe('resolve', () => { }) it('should include IPNS record in result', async () => { - const key = await generateKeyPair('Ed25519') - await name.publish(key, cid) + const keyName = 'test-key' + const { record, publicKey } = await name.publish(keyName, cid) - const customRoutingKey = multihashToIPNSRoutingKey(key.publicKey.toMultihash()) - const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) - const buf = await datastore.get(dhtKey) - const dhtRecord = Record.deserialize(buf) - const record = unmarshalIPNSRecord(dhtRecord.value) - - const result = await name.resolve(key.publicKey) + const result = await name.resolve(publicKey) - expect(result).to.have.deep.property('record', record) + expect(result).to.have.deep.property('record') + expect(marshalIPNSRecord(result.record)).to.deep.equal(marshalIPNSRecord(record)) }) it('should not search the routing for updated IPNS records when a locally cached copy is within the TTL', async () => { diff --git a/packages/ipns/test/utils.spec.ts b/packages/ipns/test/utils.spec.ts new file mode 100644 index 000000000..730bda1ee --- /dev/null +++ b/packages/ipns/test/utils.spec.ts @@ -0,0 +1,125 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { stubInterface } from 'sinon-ts' +import { shouldRepublish } from '../src/utils.ts' +import type { IPNSRecord } from '../src/index.js' + +describe('shouldRepublish', () => { + it('should return true when DHT expiry is within threshold', () => { + const now = Date.now() + const created = new Date(now - 48 * 60 * 60 * 1000 + 12 * 60 * 60 * 1000) // 36 hours ago (within 24h threshold) + const record = stubInterface({ + validity: new Date(now + 24 * 60 * 60 * 1000).toISOString() // Valid for 24 more hours + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should return true when record expiry is within threshold', () => { + const now = Date.now() + const created = new Date(now - 12 * 60 * 60 * 1000) // 12 hours ago (DHT not expired) + const record = stubInterface({ + validity: new Date(now + 12 * 60 * 60 * 1000).toISOString() // Valid for only 12 more hours (within 24h threshold) + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should return false when both DHT and record expiry are beyond threshold', () => { + const now = Date.now() + const created = new Date(now - 12 * 60 * 60 * 1000) // 12 hours ago + const record = stubInterface({ + validity: new Date(now + 36 * 60 * 60 * 1000).toISOString() // Valid for 36 more hours + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.false() + }) + + it('should return true when both expiries are within threshold', () => { + const now = Date.now() + const created = new Date(now - 36 * 60 * 60 * 1000) // 36 hours ago (DHT within threshold) + const record = stubInterface({ + validity: new Date(now + 12 * 60 * 60 * 1000).toISOString() // Valid for 12 more hours (record within threshold) + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should handle edge case with very old DHT record', () => { + const now = Date.now() + const created = new Date(now - 72 * 60 * 60 * 1000) // 72 hours ago (well past DHT expiry) + const record = stubInterface({ + validity: new Date(now + 48 * 60 * 60 * 1000).toISOString() // Valid for 48 more hours + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should handle edge case with expired record', () => { + const now = Date.now() + const created = new Date(now - 12 * 60 * 60 * 1000) // 12 hours ago + const record = stubInterface({ + validity: new Date(now - 1 * 60 * 60 * 1000).toISOString() // Expired 1 hour ago + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should work with string date format from IPNS record', () => { + const now = Date.now() + const created = new Date(now - 12 * 60 * 60 * 1000) // 12 hours ago + const record = stubInterface({ + validity: new Date(now + 12 * 60 * 60 * 1000).toISOString() // 12 hours from now (within threshold) + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should handle boundary conditions around 24 hour threshold', () => { + const now = Date.now() + + // Test just under threshold (should not republish) + const createdJustUnder = new Date(now - 23 * 60 * 60 * 1000) // 23 hours ago + const recordJustUnder = stubInterface({ + validity: new Date(now + 25 * 60 * 60 * 1000).toISOString() // Valid for 25 more hours + }) + expect(shouldRepublish(recordJustUnder, createdJustUnder)).to.be.false() + + // Test just over threshold (should republish) + const createdJustOver = new Date(now - 25 * 60 * 60 * 1000) // 25 hours ago + const recordJustOver = stubInterface({ + validity: new Date(now + 25 * 60 * 60 * 1000).toISOString() // Valid for 25 more hours + }) + expect(shouldRepublish(recordJustOver, createdJustOver)).to.be.true() + }) + + it('should return true for already expired records', () => { + const now = Date.now() + const created = new Date(now - 6 * 60 * 60 * 1000) // 6 hours ago (DHT still valid) + const record = stubInterface({ + validity: new Date(now - 3 * 60 * 60 * 1000).toISOString() // Expired 3 hours ago (recordExpiry - now is negative) + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) + + it('should return true for records that expired long ago', () => { + const now = Date.now() + const created = new Date(now - 12 * 60 * 60 * 1000) // 12 hours ago (DHT still valid) + const record = stubInterface({ + validity: new Date(now - 48 * 60 * 60 * 1000).toISOString() // Expired 48 hours ago (very negative value) + }) + + const result = shouldRepublish(record, created) + expect(result).to.be.true() + }) +}) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 983edd379..6686b7446 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -5,7 +5,7 @@ * modules such as `helia`, `@helia/http`, etc. */ -import { contentRoutingSymbol, peerRoutingSymbol, start, stop } from '@libp2p/interface' +import { contentRoutingSymbol, peerRoutingSymbol, start, stop, TypedEventEmitter } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { dns } from '@multiformats/dns' import drain from 'it-drain' @@ -18,7 +18,7 @@ import { getCodec } from './utils/get-codec.js' import { getHasher } from './utils/get-hasher.js' import { NetworkedStorage } from './utils/networked-storage.js' import type { BlockStorageInit } from './storage.js' -import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, Routing } from '@helia/interface' +import type { Await, CodecLoader, GCOptions, HasherLoader, Helia as HeliaInterface, HeliaEvents, Routing } from '@helia/interface' import type { BlockBroker } from '@helia/interface/blocks' import type { Pins } from '@helia/interface/pins' import type { ComponentLogger, Libp2p, Logger, Metrics } from '@libp2p/interface' @@ -187,6 +187,7 @@ export class Helia implements HeliaInterface { public libp2p: T public blockstore: BlockStorage public datastore: Datastore + public events: TypedEventEmitter> public pins: Pins public logger: ComponentLogger public routing: Routing @@ -204,6 +205,7 @@ export class Helia implements HeliaInterface { this.dns = init.dns ?? dns() this.metrics = init.metrics this.libp2p = init.libp2p + this.events = new TypedEventEmitter>() // @ts-expect-error routing is not set const components: Components = { @@ -261,6 +263,7 @@ export class Helia implements HeliaInterface { this.routing, this.libp2p ) + this.events.dispatchEvent(new CustomEvent('start', { detail: this })) } async stop (): Promise { @@ -270,6 +273,7 @@ export class Helia implements HeliaInterface { this.routing, this.libp2p ) + this.events.dispatchEvent(new CustomEvent('stop', { detail: this })) } async gc (options: GCOptions = {}): Promise {