Transaction: 8k5cnkB...yxPk

{ "function_name": "receiver_blocks", "code": "\n const overallTime = Date.now();\n\n // Gets index of first bit in byte array starting from startBit\n function indexOfFirstBitInByteArray(bytes, startBit) {\n let firstBit = startBit % 8;\n for (let iByte = ~~(startBit / 8); iByte < bytes.length; iByte++) {\n if (bytes[iByte] > 0) {\n for (let iBit = firstBit; iBit <= 7; iBit++) {\n if (bytes[iByte] & (1 << (7 - iBit))) {\n return iByte * 8 + iBit;\n }\n }\n }\n firstBit = 0;\n }\n return -1;\n }\n\n // Sets bit at bitIndex in uint8Array to bitValue\n function setBitInBitmap(uint8Array, bitIndex, bitValue, writeZero = false) {\n const newLen = (bitIndex / 8 + 1) >> 0; // Equivalent of Math.ciel using bitwise operations\n let result = uint8Array;\n if (uint8Array.length < newLen) {\n result = new Uint8Array(new ArrayBuffer(newLen));\n result.set(uint8Array);\n }\n if (!bitValue && writeZero) {\n result[~~(bitIndex / 8)] &= ~(1 << (7 - (bitIndex % 8)));\n } else if (bitValue) {\n result[~~(bitIndex / 8)] |= 1 << (7 - (bitIndex % 8));\n }\n return result;\n }\n\n // Gets bit at bitIndex in bytes\n function getBitInByteArray(bytes, bitIndex) {\n const byteIndex = ~~(bitIndex / 8);\n const bitIndexInsideByte = bitIndex % 8;\n return (bytes[byteIndex] & (1 << (7 - bitIndexInsideByte))) > 0;\n }\n\n // takes numbers between [start, end] bits inclusive in byte array and\n // returns decimal number they represent\n function getNumberBetweenBits(bytes, start, end) {\n let result = 0;\n // Read bits from right to left\n for (let bitIndex = end; bitIndex >= start; bitIndex--) {\n if (getBitInByteArray(bytes, bitIndex)) {\n result |= 1 << (end - bitIndex);\n }\n }\n return result;\n }\n\n // Writes Elias gamma coding bits for number x into result bytes array starting with index startBit.\n // Returns index of the next bit after the coding.\n // Examples: https://en.wikipedia.org/wiki/Elias_gamma_coding\n function writeEliasGammaBits(x, result, startBit, writeZeros = false) {\n if (x === 0) return { nextBit: startBit, result };\n if (x === 1) {\n setBitInBitmap(result, startBit, true);\n return { nextBit: startBit + 1, result };\n }\n let nextBit = startBit;\n const N = ~~Math.log2(x);\n const remainder = x - 2 ** N;\n nextBit += N;\n\n // Write N zeros if writeZeros is true\n for (let index = startBit; writeZeros && index < nextBit; index++) {\n result = setBitInBitmap(result, index, 0, writeZeros);\n }\n // Write 1 to separate prefix from suffix\n result = setBitInBitmap(result, nextBit++, true, writeZeros);\n // Write remainder in binary\n for (let ri = 0; ri < N; ri++, nextBit++) {\n const bitValue = (remainder & (1 << (N - 1 - ri))) > 0;\n result = setBitInBitmap(result, nextBit, bitValue, writeZeros);\n }\n return { nextBit, result };\n }\n\n // stores first char (0 or 1) and then repeats alternating repeating sequences using Elias gamma coding\n // pads the resulting string at the end with '0's for length to be divisible by 8\n function compressBitmapArray(uint8Array) {\n let curBit = (uint8Array[0] & 0b10000000) > 0;\n let curBitStretch = 0;\n const resultBuffer = new ArrayBuffer(12000);\n let result = new Uint8Array(resultBuffer);\n let nextBit = 0;\n let maxIndex = 0;\n\n // Write first bit to curBit\n result = setBitInBitmap(result, nextBit++, curBit);\n\n // Write Elias gamma for alternating stretches of 1s and 0s\n let lastEliasGammaStartBit = nextBit;\n let ibit = 0;\n for (; ibit < uint8Array.length * 8; ibit++) {\n if (getBitInByteArray(uint8Array, ibit) === curBit) {\n curBitStretch++;\n } else {\n maxIndex = curBit ? ibit - 1 : maxIndex;\n lastEliasGammaStartBit = curBit ? nextBit : lastEliasGammaStartBit;\n const w = writeEliasGammaBits(curBitStretch, result, nextBit);\n nextBit = w.nextBit;\n result = w.result;\n curBit = !curBit;\n curBitStretch = 1;\n }\n }\n // Write Elias Gamma for the last stretch\n maxIndex = curBit ? ibit - 1 : maxIndex;\n lastEliasGammaStartBit = curBit ? nextBit : lastEliasGammaStartBit;\n if (curBit) {\n const w = writeEliasGammaBits(curBitStretch, result, nextBit);\n nextBit = w.nextBit;\n result = w.result;\n }\n result = result.slice(0, Math.ceil(nextBit / 8));\n\n return {\n result,\n lastEliasGammaStartBit,\n lastEliasGammaBitValue: curBit,\n maxIndex,\n };\n }\n\n // Returns first number x and corresponding coded bits length of the first occurrence of Elias gamma coding\n function decodeEliasGammaEntryFromBytes(bytes, startBit = 0) {\n if (!bytes || bytes.length === 0) return { x: 0, lastBit: 0 };\n const idx = indexOfFirstBitInByteArray(bytes, startBit);\n if (idx < 0) {\n return { x: 0, lastBit: -1 };\n }\n const N = idx - startBit;\n const remainder =\n N === 0 ? 0 : getNumberBetweenBits(bytes, idx + 1, idx + N);\n return { x: 2 ** N + remainder, lastBit: idx + N };\n }\n\n // Decompresses Elias-gamma coded bytes to Uint8Array\n function decompressToBitmapArray(compressedBytes) {\n const compressedBitLength = compressedBytes.length * 8;\n let curBit = (compressedBytes[0] & 0b10000000) > 0;\n const buffer = new ArrayBuffer(11000);\n const result = new Uint8Array(buffer);\n let compressedBitIdx = 1;\n let resultBitIdx = 0;\n while (compressedBitIdx < compressedBitLength) {\n // Get x, the number of bits to set, and lastBit, the bit number which is the last bit of the Elias gamma coding\n const { x, lastBit } = decodeEliasGammaEntryFromBytes(\n compressedBytes,\n compressedBitIdx\n );\n compressedBitIdx = lastBit + 1; // Ensure next loop starts on next bit\n for (let i = 0; curBit && i < x; i++) {\n // Specifically if curBit is 1, set next x bits to 1\n setBitInBitmap(result, resultBitIdx + i, true);\n }\n resultBitIdx += x;\n curBit = !curBit; // Switch currBit for next iteration (counting 1s, then 0s, then 1s, etc.)\n if (x === 0) break; // we won't find any Elias gamma here, exiting\n }\n let bufferLength = Math.ceil(resultBitIdx / 8);\n return result.subarray(0, bufferLength);\n }\n\n // Adds a bit at position 'index' into the compressed bitmap by editing the compressed bitmap\n // from the bit at index lastEliasGammaStartBit\n function addIndexCompressedLast(\n compressedBase64,\n index,\n lastEliasGammaStartBit,\n maxIndex\n ) {\n const originalCompressed = Buffer.from(compressedBase64, \"base64\");\n const resultBuffer = Buffer.alloc(12000);\n originalCompressed.copy(resultBuffer);\n // let result = new Uint8Array(resultBuffer);\n let result = resultBuffer;\n // decode the last EG section\n const { x, lastBit } = decodeEliasGammaEntryFromBytes(\n originalCompressed,\n lastEliasGammaStartBit\n );\n let newLastEliasGammaStartBit = lastEliasGammaStartBit;\n let cur;\n if (index - maxIndex === 1) {\n // write increased stretch of 1s\n cur = writeEliasGammaBits(x + 1, result, lastEliasGammaStartBit, true);\n } else if (index - maxIndex > 1) {\n // write zeros\n const zeros = index - maxIndex - 1;\n cur = writeEliasGammaBits(zeros, result, lastBit + 1, true);\n // write 1 for the `index` bit\n newLastEliasGammaStartBit = cur.nextBit;\n cur = writeEliasGammaBits(1, cur.result, cur.nextBit, true);\n } else {\n throw Error(\n `addIndexCompressedLast: cannot write into ${index} before ${maxIndex}`\n );\n }\n // write remaining zeros\n const lastEliasGammaEndBit = cur.nextBit - 1;\n const remainingZeros = 8 - (lastEliasGammaEndBit % 8) - 1;\n for (let i = 0; i < remainingZeros; i++) {\n result = setBitInBitmap(cur.result, cur.nextBit + i, false, true);\n }\n\n return {\n compressed: result\n .subarray(0, Math.ceil(cur.nextBit / 8))\n .toString(\"base64\"),\n lastEliasGammaStartBit: newLastEliasGammaStartBit,\n maxIndex: index,\n };\n }\n\n function addIndexCompressedFull(compressedBase64, index) {\n const buf = Buffer.from(compressedBase64, \"base64\");\n const bitmap = decompressToBitmapArray(buf);\n const newBitmap = setBitInBitmap(bitmap, index, true);\n const {\n result: compressed,\n lastEliasGammaStartBit,\n maxIndex,\n } = compressBitmapArray(newBitmap);\n return {\n compressed: Buffer.from(compressed).toString(\"base64\"),\n lastEliasGammaStartBit,\n maxIndex,\n };\n }\n\n function addIndexCompressed(\n compressedBase64,\n index,\n lastEliasGammaStartBit,\n maxIndex = -1\n ) {\n if (maxIndex < 0 || index <= maxIndex) {\n return addIndexCompressedFull(compressedBase64, index);\n } else {\n return addIndexCompressedLast(\n compressedBase64,\n index,\n lastEliasGammaStartBit,\n maxIndex\n );\n }\n }\n\n const blockDate = new Date(\n block.streamerMessage.block.header.timestamp / 1000000\n )\n .toISOString()\n .substring(0, 10);\n const actionsByReceiver = block.actions().reduce((groups, action) => {\n (groups[action.receiverId] ||= []).push(action);\n return groups;\n }, {});\n let allReceivers = Array.from(\n Object.keys(actionsByReceiver).reduce((totalReceivers, receiver) => {\n totalReceivers.add(receiver);\n const lastIndexOfPeriod = receiver.lastIndexOf(\".\");\n return lastIndexOfPeriod === -1\n ? totalReceivers.add(receiver)\n : totalReceivers.add(receiver.substring(lastIndexOfPeriod + 1));\n }, new Set())\n );\n const receiverMap = (\n await context.db.Receivers.upsert(\n allReceivers.map((r) => ({ receiver: r })),\n [\"receiver\"],\n [\"receiver\"]\n )\n ).reduce((map, row) => ({ ...map, [row.receiver]: row.id }), {});\n //context.log('receiverMap', JSON.stringify(receiverMap));\n const existingReceivers =\n (await context.db.Bitmaps.select({\n block_date: blockDate,\n receiver_id: Object.values(receiverMap),\n })) ?? [];\n\n let startTime = Date.now();\n const upserts = Object.entries(receiverMap).map(([receiver, receiverId]) => {\n const currentIndex = existingReceivers.find(\n (receiver) => receiver?.receiver_id === receiverId\n );\n const blockIndexInCurrentBitmap = currentIndex?.first_block_height\n ? block.blockHeight - currentIndex?.first_block_height\n : 0;\n\n const newBitmap = addIndexCompressed(\n currentIndex?.bitmap ?? \"\",\n blockIndexInCurrentBitmap,\n parseInt(currentIndex?.last_elias_gamma_start_bit ?? -1),\n parseInt(currentIndex?.max_index ?? -1)\n );\n return {\n first_block_height: currentIndex?.first_block_height ?? block.blockHeight,\n block_date: blockDate,\n receiver_id: receiverId,\n bitmap: newBitmap.compressed,\n last_elias_gamma_start_bit: newBitmap.lastEliasGammaStartBit,\n max_index: newBitmap.maxIndex,\n };\n });\n const processBitmapsTimer = Date.now() - startTime;\n\n await context.db.Bitmaps.upsert(\n upserts,\n [\"block_date\", \"receiver_id\"],\n [\"bitmap\", \"last_elias_gamma_start_bit\", \"max_index\"]\n );\n console.log(\n `Computing bitmaps for ${\n allReceivers.length\n } receivers took ${processBitmapsTimer}ms; total time ${\n Date.now() - overallTime\n }ms`\n );\n", "schema": "CREATE TABLE\n \"receivers\" (\n \"id\" BIGSERIAL NOT NULL PRIMARY KEY,\n \"receiver\" TEXT NOT NULL\n );\n\nCREATE UNIQUE INDEX idx_receivers_by_receiver ON receivers (receiver);\n\nCREATE TABLE\n \"bitmaps\" (\n \"receiver_id\" bigint NOT NULL,\n \"block_date\" date NOT NULL,\n \"first_block_height\" int NOT NULL,\n \"last_elias_gamma_start_bit\" int NOT NULL,\n \"max_index\" int NOT NULL,\n \"bitmap\" TEXT NOT NULL,\n PRIMARY KEY (\"block_date\", \"receiver_id\"),\n CONSTRAINT \"bitmaps_receiver_id_fkey\" FOREIGN KEY (\"receiver_id\") REFERENCES \"receivers\" (\"id\") ON DELETE NO ACTION ON UPDATE NO ACTION\n );\n", "start_block": { "HEIGHT": 121550500 }, "rule": { "kind": "ACTION_ANY", "affected_account_id": "*", "status": "SUCCESS" } }