-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* chore: Update example, using reed solomon * chore: Update Rollup config * chore: Benchmark * docs: Update README * feat: Support WebWorker
- Loading branch information
Showing
14 changed files
with
263 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| '@bnb-chain/reed-solomon': patch | ||
| --- | ||
|
|
||
| feat: Support Web Worker |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| # benchmarks | ||
|
|
||
| ## 20M | ||
|
|
||
| * Nodejs: 1s | ||
| * Nodejs(worker): 1s | ||
|
|
||
| ## 200M | ||
|
|
||
| * Go: 300ms | ||
| * Nodejs: 10s | ||
| * Nodejs(worker): 2s | ||
| * Browser: 15s | ||
|
|
||
| ## conclusion | ||
|
|
||
| When there are not many shards, using worker will not make performance faster. | ||
|
|
||
| Because worker communication is also a performance loss. |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8"> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||
| <title>Document</title> | ||
| </head> | ||
| <body> | ||
| <input type="file" id="file" /> | ||
|
|
||
| <button id="btn"> | ||
| get reed solomon | ||
| </button> | ||
|
|
||
| <button id="worker-btn"> | ||
| get reed solomon (webworker) | ||
| </button> | ||
|
|
||
| <script src="../dist/index.aio.js"></script> | ||
| <script src="../dist/web.adapter.aio.js"></script> | ||
| <script src="../dist/utils.aio.js"></script> | ||
| <script type="module"> | ||
| const fileInput = document.getElementById('file'); | ||
|
|
||
| // not use webworker | ||
| document.getElementById('btn').onclick = async function() { | ||
| const selectFile = fileInput.files[0]; | ||
| const arrBuffer = await selectFile.arrayBuffer() | ||
| if (!arrBuffer) alert('no file selected'); | ||
|
|
||
| const sourceData = new Uint8Array(arrBuffer) | ||
| console.time('cost') | ||
| console.log('file size', sourceData.length / 1024 / 1024, 'm') | ||
| const rs = new RS.ReedSolomon() | ||
| const res = await rs.encode(sourceData) | ||
| console.log('res', res) | ||
| console.timeEnd('cost') | ||
| } | ||
|
|
||
| // use webworker | ||
| document.getElementById('worker-btn').onclick = async function() { | ||
| const selectFile = fileInput.files[0]; | ||
| const arrBuffer = await selectFile.arrayBuffer() | ||
| if (!arrBuffer) alert('no file selected'); | ||
|
|
||
| const sourceData = new Uint8Array(arrBuffer) | ||
| console.time('webworker cost') | ||
| console.log('file size', sourceData.length / 1024 / 1024, 'm') | ||
| const rs = new WebAdapter.WebAdapterReedSolomon() | ||
| const res = await rs.encodeInWorker(WebAdapter.injectWorker, sourceData) | ||
| console.log('res', res) | ||
| console.timeEnd('webworker cost') | ||
| } | ||
|
|
||
| </script> | ||
| </body> | ||
| </html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| import { ReedSolomon } from '.'; | ||
| import { sha256, getIntegrityUint8Array, toBase64, splitPrice } from './utils'; | ||
|
|
||
| export class WebAdapterReedSolomon extends ReedSolomon { | ||
| async encodeInWorker(workerFn, sourceData, { webAdapterUrl, utilsUrl }) { | ||
| const chunkList = splitPrice(sourceData, this.segmentSize); | ||
|
|
||
| const workers = []; | ||
|
|
||
| for (let i = 0; i < chunkList.length; i++) { | ||
| // const worker = new Worker('worker.js'); | ||
| const worker = createWorker(workerFn, { | ||
| webAdapterUrl, | ||
| utilsUrl, | ||
| }); | ||
| workers.push(worker); | ||
| worker.postMessage({ | ||
| index: i, | ||
| chunk: chunkList[i], | ||
| }); | ||
| } | ||
|
|
||
| const plist = workers.map( | ||
| (worker) => | ||
| new Promise((resolve) => { | ||
| worker.onmessage = (e) => { | ||
| resolve(e.data); | ||
| }; | ||
| }), | ||
| ); | ||
|
|
||
| return Promise.all(plist).then((RES) => { | ||
| let hashList = []; | ||
| let segChecksumList = []; | ||
| let encodeDataHashList = new Array(this.totalShards); | ||
| for (let i = 0; i < encodeDataHashList.length; i++) { | ||
| encodeDataHashList[i] = []; | ||
| } | ||
|
|
||
| for (let i = 0; i < RES.length; i++) { | ||
| segChecksumList.push(RES[i].segChecksum); | ||
| } | ||
|
|
||
| for (let i = 0; i < chunkList.length; i++) { | ||
| for (let j = 0; j < encodeDataHashList.length; j++) { | ||
| encodeDataHashList[j][i] = RES[i].encodeDataHash[j]; | ||
| } | ||
| } | ||
|
|
||
| hashList[0] = sha256(getIntegrityUint8Array(segChecksumList)); | ||
|
|
||
| for (let i = 0; i < encodeDataHashList.length; i++) { | ||
| hashList[i + 1] = sha256(getIntegrityUint8Array(encodeDataHashList[i])); | ||
| } | ||
|
|
||
| const res = toBase64(hashList); | ||
|
|
||
| return res; | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| function createWorker(f, { webAdapterUrl, utilsUrl }) { | ||
| var blob = new Blob([ | ||
| '(' + f.toString() + ')(' + `'${webAdapterUrl}'` + ',' + `'${utilsUrl}'` + ')', | ||
| ]); | ||
| var url = window.URL.createObjectURL(blob); | ||
| var worker = new Worker(url); | ||
| return worker; | ||
| } | ||
|
|
||
| // inject worker script | ||
| export function injectWorker(cdnsUrls) { | ||
| importScripts( | ||
| cdnsUrls.webAdapterUrl || | ||
| 'https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/index.aio.js', | ||
| ); | ||
| importScripts('https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/utils.aio.js'); | ||
|
|
||
| const rs = new WebAdapter.WebAdapterReedSolomon(); | ||
|
|
||
| onmessage = function (event) { | ||
| const { index, chunk } = event.data; | ||
| const encodeShards = rs.encodeSegment(chunk); | ||
| let encodeDataHash = []; | ||
|
|
||
| for (let i = 0; i < encodeShards.length; i++) { | ||
| const priceHash = RSUtils.sha256(encodeShards[i]); | ||
| encodeDataHash.push(priceHash); | ||
| } | ||
|
|
||
| postMessage({ | ||
| index, | ||
| segChecksum: RSUtils.sha256(chunk), | ||
| encodeDataHash, | ||
| }); | ||
|
|
||
| self.close(); | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| declare module '@bnb-chain/reed-solomon/web.adapter' { | ||
| export class WebAdapterReedSolomon { | ||
| encodeInWorker( | ||
| workerFn: () => void, | ||
| data: Uint8Array, | ||
| cdnUrls: { | ||
| webAdapterUrl: string; | ||
| utilsUrl: string; | ||
| }, | ||
| ): Promise<string[]>; | ||
| } | ||
|
|
||
| export function injectWorker(cdnsUrls?: { webAdapterUrl: string; utilsUrl: string }): void; | ||
| } |
Oops, something went wrong.