From afb54644683c8da508d250731a6462676cf44747 Mon Sep 17 00:00:00 2001 From: rk <59029880+rrr523@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:53:45 +0800 Subject: [PATCH] Feat/rs webworker (#429) * chore: Update example, using reed solomon * chore: Update Rollup config * chore: Benchmark * docs: Update README * feat: Support WebWorker --- .changeset/dirty-buttons-care.md | 5 + examples/nextjs/package.json | 1 + .../src/components/object/create/index.tsx | 13 +-- packages/reed-solomon/README.md | 2 +- packages/reed-solomon/benchmark.md | 19 ++++ packages/reed-solomon/examples/singlefile.js | 16 --- packages/reed-solomon/examples/web.html | 57 ++++++++++ packages/reed-solomon/package.json | 17 ++- packages/reed-solomon/rollup.config.js | 43 +++++++- packages/reed-solomon/src/utils.js | 1 + packages/reed-solomon/src/web.adapter.js | 100 ++++++++++++++++++ packages/reed-solomon/types/index.d.ts | 4 - packages/reed-solomon/types/web.adapter.d.ts | 14 +++ pnpm-lock.yaml | 3 + 14 files changed, 263 insertions(+), 32 deletions(-) create mode 100644 .changeset/dirty-buttons-care.md create mode 100644 packages/reed-solomon/benchmark.md delete mode 100644 packages/reed-solomon/examples/singlefile.js create mode 100644 packages/reed-solomon/examples/web.html create mode 100644 packages/reed-solomon/src/web.adapter.js create mode 100644 packages/reed-solomon/types/web.adapter.d.ts diff --git a/.changeset/dirty-buttons-care.md b/.changeset/dirty-buttons-care.md new file mode 100644 index 00000000..b2cbcb65 --- /dev/null +++ b/.changeset/dirty-buttons-care.md @@ -0,0 +1,5 @@ +--- +'@bnb-chain/reed-solomon': patch +--- + +feat: Support Web Worker diff --git a/examples/nextjs/package.json b/examples/nextjs/package.json index 1fe739d5..a3dd3282 100644 --- a/examples/nextjs/package.json +++ b/examples/nextjs/package.json @@ -14,6 +14,7 @@ "@bnb-chain/greenfield-cosmos-types": "0.4.0-alpha.25", "@bnb-chain/greenfield-js-sdk": "workspace:*", "@bnb-chain/greenfiled-file-handle": "workspace:*", + "@bnb-chain/reed-solomon": "workspace:*", "@cosmjs/encoding": "^0.32.0", "@cosmjs/proto-signing": "^0.32.0", "@cosmjs/stargate": "^0.32.0", diff --git a/examples/nextjs/src/components/object/create/index.tsx b/examples/nextjs/src/components/object/create/index.tsx index ee9a8efb..9427665a 100644 --- a/examples/nextjs/src/components/object/create/index.tsx +++ b/examples/nextjs/src/components/object/create/index.tsx @@ -1,9 +1,9 @@ import { client } from '@/client'; -import { ACCOUNT_PRIVATEKEY } from '@/config/env'; import { getOffchainAuthKeys } from '@/utils/offchainAuth'; import { ChangeEvent, useState } from 'react'; import { useAccount } from 'wagmi'; import { getCheckSumsWorker } from '@bnb-chain/greenfiled-file-handle'; +import { ReedSolomon } from '@bnb-chain/reed-solomon'; export const CreateObject = () => { const { address, connector } = useAccount(); @@ -64,14 +64,11 @@ export const CreateObject = () => { const multiCal = await checksumWorker.generateCheckSumV2(file); console.log('multiCal', multiCal); + const rs = new ReedSolomon(); const fileBytes = await file.arrayBuffer(); - const hashResult = await (window as any).FileHandle.getCheckSums( - new Uint8Array(fileBytes), - ); - const { contentLength, expectCheckSums } = hashResult; + const expectCheckSums = rs.encode(new Uint8Array(fileBytes)); console.log('offChainData', offChainData); - console.log('hashResult', hashResult); const createObjectTx = await client.object.createObject( { @@ -81,8 +78,8 @@ export const CreateObject = () => { visibility: 'VISIBILITY_TYPE_PRIVATE', fileType: file.type, redundancyType: 'REDUNDANCY_EC_TYPE', - contentLength, - expectCheckSums: JSON.parse(expectCheckSums), + contentLength: fileBytes.byteLength, + expectCheckSums: expectCheckSums, // empty tags // tags: { // tags: [], diff --git a/packages/reed-solomon/README.md b/packages/reed-solomon/README.md index 43c03890..647d9d0a 100644 --- a/packages/reed-solomon/README.md +++ b/packages/reed-solomon/README.md @@ -22,7 +22,7 @@ Use directly in the browser via script tag: get reed solomon - + + + + + + diff --git a/packages/reed-solomon/package.json b/packages/reed-solomon/package.json index c6ff6d92..efec1ab1 100644 --- a/packages/reed-solomon/package.json +++ b/packages/reed-solomon/package.json @@ -7,7 +7,7 @@ "types": "./types/index.d.ts", "exports": { ".": { - "import": "./dist/index.aio.js", + "import": "./dist/index.js", "require": "./dist/index.js", "main": "./dist/index.js", "default": "./dist/index.js", @@ -19,11 +19,26 @@ "node": "./dist/node.adapter.js", "require": "./dist/node.adapter.js" }, + "./web.adapter": { + "default": "./dist/web.adapter.js", + "types": "./types/web.adapter.d.ts", + "import": "./dist/web.adapter.js" + }, "./utils": { + "import": "./dist/utils.esm.js", + "require": "./dist/utils.js", + "main": "./dist/utils.js", "default": "./dist/utils.js", "types": "./types/utils.d.ts" } }, + "typesVersions": { + "*": { + "node.adapter": ["./types/node.adapter.d.ts"], + "web.adapter": ["./types/web.adapter.d.ts"], + "utils": ["./types/utils.d.ts"] + } + }, "scripts": { "prebuild": "rimraf ./dist", "build": "rollup -c" diff --git a/packages/reed-solomon/rollup.config.js b/packages/reed-solomon/rollup.config.js index cc58f722..d7f4349d 100644 --- a/packages/reed-solomon/rollup.config.js +++ b/packages/reed-solomon/rollup.config.js @@ -9,10 +9,13 @@ function resolveExternal() { export default async () => { return [ + // ESM { - input: './src/index.js', + input: ['./src/index.js', './src/web.adapter.js'], output: { - file: 'dist/index.esm.js', + format: 'es', + dir: 'dist', + entryFileNames: '[name].esm.js', sourcemap: true, }, external: resolveExternal(), @@ -26,6 +29,8 @@ export default async () => { }), ], }, + + // CJS { input: ['./src/index.js', './src/node.adapter.js', './src/utils.js'], output: { @@ -43,6 +48,8 @@ export default async () => { // }), ], }, + + // UMD { input: './src/index.js', output: { @@ -59,5 +66,37 @@ export default async () => { }), ], }, + { + input: './src/web.adapter.js', + output: { + format: 'umd', + file: 'dist/web.adapter.aio.js', + name: 'WebAdapter', + sourcemap: true, + }, + plugins: [ + commonjs(), + resolve({ + browser: true, + preferBuiltins: false, + }), + ], + }, + { + input: './src/utils.js', + output: { + format: 'umd', + file: 'dist/utils.aio.js', + name: 'RSUtils', + sourcemap: true, + }, + plugins: [ + commonjs(), + resolve({ + browser: true, + preferBuiltins: false, + }), + ], + }, ]; }; diff --git a/packages/reed-solomon/src/utils.js b/packages/reed-solomon/src/utils.js index 55f81d2b..91fcdc7f 100644 --- a/packages/reed-solomon/src/utils.js +++ b/packages/reed-solomon/src/utils.js @@ -1,4 +1,5 @@ import { encode } from '@ethersproject/base64'; +export { sha256 } from 'ethereum-cryptography/sha256.js'; export function concat(a, b) { let res = []; diff --git a/packages/reed-solomon/src/web.adapter.js b/packages/reed-solomon/src/web.adapter.js new file mode 100644 index 00000000..9374b1ea --- /dev/null +++ b/packages/reed-solomon/src/web.adapter.js @@ -0,0 +1,100 @@ +import { ReedSolomon } from '.'; +import { sha256, getIntegrityUint8Array, toBase64, splitPrice } from './utils'; + +export class WebAdapterReedSolomon extends ReedSolomon { + async encodeInWorker(workerFn, sourceData, { webAdapterUrl, utilsUrl }) { + const chunkList = splitPrice(sourceData, this.segmentSize); + + const workers = []; + + for (let i = 0; i < chunkList.length; i++) { + // const worker = new Worker('worker.js'); + const worker = createWorker(workerFn, { + webAdapterUrl, + utilsUrl, + }); + workers.push(worker); + worker.postMessage({ + index: i, + chunk: chunkList[i], + }); + } + + const plist = workers.map( + (worker) => + new Promise((resolve) => { + worker.onmessage = (e) => { + resolve(e.data); + }; + }), + ); + + return Promise.all(plist).then((RES) => { + let hashList = []; + let segChecksumList = []; + let encodeDataHashList = new Array(this.totalShards); + for (let i = 0; i < encodeDataHashList.length; i++) { + encodeDataHashList[i] = []; + } + + for (let i = 0; i < RES.length; i++) { + segChecksumList.push(RES[i].segChecksum); + } + + for (let i = 0; i < chunkList.length; i++) { + for (let j = 0; j < encodeDataHashList.length; j++) { + encodeDataHashList[j][i] = RES[i].encodeDataHash[j]; + } + } + + hashList[0] = sha256(getIntegrityUint8Array(segChecksumList)); + + for (let i = 0; i < encodeDataHashList.length; i++) { + hashList[i + 1] = sha256(getIntegrityUint8Array(encodeDataHashList[i])); + } + + const res = toBase64(hashList); + + return res; + }); + } +} + +function createWorker(f, { webAdapterUrl, utilsUrl }) { + var blob = new Blob([ + '(' + f.toString() + ')(' + `'${webAdapterUrl}'` + ',' + `'${utilsUrl}'` + ')', + ]); + var url = window.URL.createObjectURL(blob); + var worker = new Worker(url); + return worker; +} + +// inject worker script +export function injectWorker(cdnsUrls) { + importScripts( + cdnsUrls.webAdapterUrl || + 'https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/index.aio.js', + ); + importScripts('https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/utils.aio.js'); + + const rs = new WebAdapter.WebAdapterReedSolomon(); + + onmessage = function (event) { + const { index, chunk } = event.data; + const encodeShards = rs.encodeSegment(chunk); + let encodeDataHash = []; + + for (let i = 0; i < encodeShards.length; i++) { + const priceHash = RSUtils.sha256(encodeShards[i]); + encodeDataHash.push(priceHash); + } + + postMessage({ + index, + segChecksum: RSUtils.sha256(chunk), + encodeDataHash, + }); + + self.close(); + }; +} diff --git a/packages/reed-solomon/types/index.d.ts b/packages/reed-solomon/types/index.d.ts index 1a0251a5..abff184a 100644 --- a/packages/reed-solomon/types/index.d.ts +++ b/packages/reed-solomon/types/index.d.ts @@ -7,7 +7,3 @@ declare module '@bnb-chain/reed-solomon' { encode(data: Uint8Arary): string[]; } } - -// declare class NodeAdapterReedSolomon { -// encodeInWorker(p: string, data: Uint8Array): Promise; -// } diff --git a/packages/reed-solomon/types/web.adapter.d.ts b/packages/reed-solomon/types/web.adapter.d.ts new file mode 100644 index 00000000..e965106a --- /dev/null +++ b/packages/reed-solomon/types/web.adapter.d.ts @@ -0,0 +1,14 @@ +declare module '@bnb-chain/reed-solomon/web.adapter' { + export class WebAdapterReedSolomon { + encodeInWorker( + workerFn: () => void, + data: Uint8Array, + cdnUrls: { + webAdapterUrl: string; + utilsUrl: string; + }, + ): Promise; + } + + export function injectWorker(cdnsUrls?: { webAdapterUrl: string; utilsUrl: string }): void; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9bc82386..ddc78440 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -147,6 +147,9 @@ importers: '@bnb-chain/greenfiled-file-handle': specifier: workspace:* version: link:../../packages/file-handle + '@bnb-chain/reed-solomon': + specifier: workspace:* + version: link:../../packages/reed-solomon '@cosmjs/encoding': specifier: ^0.32.0 version: 0.32.0