fix(cbor): match byte-string map keys when replaying a decoded map

.changeset/data-map-bytes-key.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+A decoded transaction now keeps the original bytes of witness datums and redeemers that contain a Plutus data map whose keys hold byte strings. This covers byte-string keys, list and constructor keys that contain them, and repeated keys. Before, re-encoding the transaction or adding a witness with `Transaction.addVKeyWitnessesHex` rewrote the value under each such key, so the node reported a script data hash mismatch.

packages/evolution-devnet/test/Transaction.RedeemerMapBytesKey.test.ts

@@ -0,0 +1,191 @@
+/**
+ * Devnet test: decoding a transaction and adding a witness keeps the original
+ * bytes of a redeemer that holds a Plutus data map with a byte-string key.
+ *
+ * The node hashes the redeemer bytes as written into the script data hash. A
+ * script spend whose redeemer is {h'01': [1, 2]} with an indefinite list is
+ * signed over its raw body, merged with Transaction.addVKeyWitnessesBytes and
+ * submitted as raw bytes. If the merge rewrites the list as definite, the node
+ * rejects the transaction with a script data hash mismatch.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest"
+import * as Cluster from "@evolution-sdk/devnet/Cluster"
+import * as Config from "@evolution-sdk/devnet/Config"
+import * as Genesis from "@evolution-sdk/devnet/Genesis"
+import { Cardano, Client, preprod } from "@evolution-sdk/evolution"
+import * as Address from "@evolution-sdk/evolution/Address"
+import * as Bytes from "@evolution-sdk/evolution/Bytes"
+import * as CBOR from "@evolution-sdk/evolution/CBOR"
+import * as CostModel from "@evolution-sdk/evolution/CostModel"
+import * as PlutusData from "@evolution-sdk/evolution/Data"
+import * as InlineDatum from "@evolution-sdk/evolution/InlineDatum"
+import * as PlutusV3 from "@evolution-sdk/evolution/PlutusV3"
+import * as ScriptHash from "@evolution-sdk/evolution/ScriptHash"
+import * as Transaction from "@evolution-sdk/evolution/Transaction"
+import * as TransactionBody from "@evolution-sdk/evolution/TransactionBody"
+import * as TransactionHash from "@evolution-sdk/evolution/TransactionHash"
+import * as TransactionWitnessSet from "@evolution-sdk/evolution/TransactionWitnessSet"
+import { blake2b } from "@noble/hashes/blake2.js"
+
+describe("Redeemer map with byte-string key (Devnet Submit)", () => {
+  let devnetCluster: Cluster.Cluster | undefined
+  let genesisUtxos: ReadonlyArray<Cardano.UTxO.UTxO> = []
+
+  const TEST_MNEMONIC =
+    "test test test test test test test test test test test test test test test test test test test test test test test sauce"
+
+  // always_succeed.always_succeed.spend from packages/evolution/test/spec
+  const ALWAYS_SUCCEED_COMPILED_CODE =
+    "587e01010029800aba2aba1aab9eaab9dab9cab9a48888896600264653001300700198039804000cc01c0092225980099b8748008c020dd500144c8cc892898058009805980600098049baa0028a50401830070013004375400f149a2a660049211856616c696461746f722072657475726e65642066616c7365001365640041"
+
+  const alwaysSucceedScript = new PlutusV3.PlutusV3({ bytes: Bytes.fromHex(ALWAYS_SUCCEED_COMPILED_CODE) })
+  const scriptAddress = Address.Address.make({
+    networkId: 0,
+    paymentCredential: ScriptHash.fromScript(alwaysSucceedScript)
+  })
+
+  const ogmiosUrl = () => `http://localhost:${devnetCluster!.ports.ogmios}`
+
+  const createTestClient = () => {
+    if (!devnetCluster) throw new Error("Cluster not initialized")
+    return Client.make(Cluster.getChain(devnetCluster))
+      .withKupmios({ kupoUrl: `http://localhost:${devnetCluster.ports.kupo}`, ogmiosUrl: ogmiosUrl() })
+      .withSeed({ mnemonic: TEST_MNEMONIC, accountIndex: 0, addressType: "Base" })
+  }
+
+  const ogmios = async (method: string, params: unknown) => {
+    const response = await fetch(ogmiosUrl(), {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ jsonrpc: "2.0", method, params, id: null })
+    })
+    return (await response.json()) as { result?: any; error?: unknown }
+  }
+
+  // Raw bytes of the witness set redeemers (key 5), as the node hashes them
+  const redeemersBytes = (txBytes: Uint8Array): Uint8Array => {
+    const witnessSetStart = CBOR.decodeItemWithOffset(txBytes, 1).newOffset
+    // The witness set is a definite map with fewer than 24 entries
+    const entryCount = txBytes[witnessSetStart] - 0xa0
+    let offset = witnessSetStart + 1
+    for (let i = 0; i < entryCount; i++) {
+      const key = CBOR.decodeItemWithOffset(txBytes, offset)
+      const value = CBOR.decodeItemWithOffset(txBytes, key.newOffset)
+      if (key.item === 5n) return txBytes.slice(key.newOffset, value.newOffset)
+      offset = value.newOffset
+    }
+    throw new Error("No redeemers in witness set")
+  }
+
+  beforeAll(async () => {
+    const tempClient = Client.make(preprod).withSeed({ mnemonic: TEST_MNEMONIC, accountIndex: 0, addressType: "Base" })
+    const testAddressHex = Address.toHex(await tempClient.address())
+
+    const genesisConfig: Config.ShelleyGenesis = {
+      ...Config.DEFAULT_SHELLEY_GENESIS,
+      slotLength: 0.02,
+      epochLength: 50,
+      activeSlotsCoeff: 1.0,
+      initialFunds: { [testAddressHex]: 500_000_000_000 }
+    }
+
+    genesisUtxos = await Genesis.calculateUtxosFromConfig(genesisConfig)
+
+    devnetCluster = await Cluster.make({
+      clusterName: "redeemer-map-bytes-key-test",
+      shelleyGenesis: genesisConfig,
+      kupo: { enabled: true, logLevel: "Info" },
+      ogmios: { enabled: true, logLevel: "info" }
+    })
+
+    await Cluster.start(devnetCluster)
+    await new Promise((resolve) => setTimeout(resolve, 3_000))
+  }, 180_000)
+
+  afterAll(async () => {
+    if (devnetCluster) {
+      await Cluster.stop(devnetCluster)
+      await Cluster.remove(devnetCluster)
+    }
+  }, 60_000)
+
+  it("keeps an indefinite list under a byte key through addVKeyWitnessesBytes", { timeout: 120_000 }, async () => {
+    const client = createTestClient()
+    const myAddress = await client.address()
+
+    const deployBuilder = await client
+      .newTx()
+      .payToAddress({
+        address: scriptAddress,
+        assets: Cardano.Assets.fromLovelace(10_000_000n),
+        datum: new InlineDatum.InlineDatum({ data: PlutusData.int(42n) })
+      })
+      .build({ availableUtxos: [...genesisUtxos] })
+    const deployTxHash = await (await deployBuilder.sign()).submit()
+    expect(await client.awaitTx(deployTxHash, 1000)).toBe(true)
+    await new Promise((resolve) => setTimeout(resolve, 2_000))
+
+    const scriptUtxos = await client.getUtxos(scriptAddress)
+    expect(scriptUtxos.length).toBe(1)
+
+    // The builder writes {h'01': [100, 2]} with a definite list (a1 41 01 82 1864 02).
+    // It is rewritten as {h'01': [1, 2]} with an indefinite list (a1 41 01 9f 01 02 ff),
+    // the same length, so the fee still covers it. The script ignores the redeemer.
+    const builtRedeemerData = "a14101821864" + "02"
+    const redeemerData = "a141019f0102ff"
+    const signBuilder = await client
+      .newTx()
+      .collectFrom({
+        inputs: [...scriptUtxos],
+        redeemer: PlutusData.map([
+          [PlutusData.bytearray("01"), PlutusData.list([PlutusData.int(100n), PlutusData.int(2n)])]
+        ])
+      })
+      .attachScript({ script: alwaysSucceedScript })
+      .payToAddress({ address: myAddress, assets: Cardano.Assets.fromLovelace(5_000_000n) })
+      .build()
+
+    const builtTx = await signBuilder.toTransaction()
+    const builtHex = Transaction.toCBORHex(builtTx)
+    const builtScriptDataHash = Bytes.toHex(builtTx.body.scriptDataHash!.hash)
+
+    // Language views for the one PlutusV3 script, checked against the builder's own hash
+    const params = await client.getProtocolParameters()
+    const languageViews = CostModel.languageViewsEncoding(
+      new CostModel.CostModels({
+        PlutusV1: new CostModel.CostModel({ costs: [] }),
+        PlutusV2: new CostModel.CostModel({ costs: [] }),
+        PlutusV3: new CostModel.CostModel({ costs: Object.values(params.costModels.PlutusV3).map((v) => BigInt(v)) })
+      })
+    )
+    const scriptDataHash = (txBytes: Uint8Array) =>
+      Bytes.toHex(blake2b(new Uint8Array([...redeemersBytes(txBytes), ...languageViews]), { dkLen: 32 }))
+    expect(scriptDataHash(Bytes.fromHex(builtHex))).toBe(builtScriptDataHash)
+
+    expect(builtHex.split(builtRedeemerData).length).toBe(2)
+    expect(builtHex.split("0b5820" + builtScriptDataHash).length).toBe(2)
+    const rewrittenHex = builtHex.replace(builtRedeemerData, redeemerData)
+    const txHex = rewrittenHex.replace(
+      "0b5820" + builtScriptDataHash,
+      "0b5820" + scriptDataHash(Bytes.fromHex(rewrittenHex))
+    )
+    const txBytes = Bytes.fromHex(txHex)
+    const bodyBytes = Transaction.extractBodyBytes(txBytes)
+    const expectedTxId = TransactionHash.toHex(TransactionBody.toHashFromBytes(bodyBytes))
+
+    // Sign over the raw body, then merge the witness into the decoded transaction
+    const walletWitnessSet = await client.signTx(txHex, { utxos: [...genesisUtxos, ...scriptUtxos] })
+    expect(walletWitnessSet.vkeyWitnesses?.length).toBe(1)
+    const signedBytes = Transaction.addVKeyWitnessesBytes(txBytes, TransactionWitnessSet.toCBORBytes(walletWitnessSet))
+
+    const submitted = await ogmios("submitTransaction", { transaction: { cbor: Bytes.toHex(signedBytes) } })
+    expect(submitted.error).toBeUndefined()
+    expect(submitted.result.transaction.id).toBe(expectedTxId)
+    expect(Bytes.toHex(Transaction.extractBodyBytes(signedBytes))).toBe(Bytes.toHex(bodyBytes))
+    expect(Bytes.toHex(redeemersBytes(signedBytes))).toBe(Bytes.toHex(redeemersBytes(txBytes)))
+    expect(Bytes.toHex(redeemersBytes(signedBytes))).toContain(redeemerData)
+
+    expect(await client.awaitTx(TransactionHash.fromHex(expectedTxId), 1000)).toBe(true)
+  })
+})

packages/evolution/src/CBOR.ts

@@ -1522,21 +1522,16 @@ const encodeMapEntriesSync = (pairs: Array<[CBOR, CBOR]>, options: CodecOptions,
  */
 export const equals: (a: CBOR, b: CBOR) => boolean = Schema.equivalence(CBORSchema)
 
-/**
- * Look up a CBOR key in a Map, falling back to content-based comparison
- * for complex keys (Uint8Array, Array, Tag) where Map.get uses reference
- * equality which fails when the map was rebuilt with new objects.
- */
-const mapGetCBOR = (map: ReadonlyMap<CBOR, CBOR>, key: CBOR): CBOR | undefined => {
-  const direct = map.get(key)
-  if (direct !== undefined) return direct
-  // Primitives (bigint, string, boolean, null, number) match by value equality
-  // via Map.get; if that failed, the key genuinely doesn't exist
-  if (typeof key !== "object" || key === null) return undefined
-  for (const [k, v] of map) {
-    if (equals(key, k)) return v
-  }
-  return undefined
+// A Plutus data byte string is a BoundedBytes node, but the same bytes decoded
+// from keyOrder are a plain Uint8Array, at any depth of the key
+const unwrapBoundedBytes = (value: CBOR): CBOR => {
+  if (BoundedBytes.is(value)) return value.bytes
+  if (Array.isArray(value)) return value.map(unwrapBoundedBytes)
+  if (value instanceof Map) {
+    return new Map(Array.from(value, ([k, v]): [CBOR, CBOR] => [unwrapBoundedBytes(k), unwrapBoundedBytes(v)]))
+  }
+  if (isTag(value)) return { _tag: "Tag", tag: value.tag, value: unwrapBoundedBytes(value.value) }
+  return value
 }
 
 const encodeMapSync = (value: ReadonlyMap<CBOR, CBOR>, options: CodecOptions, fmt?: CBORFormat): Uint8Array => {
@@ -1545,24 +1540,29 @@ const encodeMapSync = (value: ReadonlyMap<CBOR, CBOR>, options: CodecOptions, fm
   if (mapFmt?.keyOrder && mapFmt.keyOrder.length > 0) {
     const pairs: Array<[CBOR, CBOR]> = []
     const reorderedEntries: Array<readonly [CBORFormat, CBORFormat]> = []
-    const decodedKeyOrderKeys: Array<CBOR> = []
+    const entries = Array.from(value.entries())
+    const plainKeys = entries.map(([k]) => unwrapBoundedBytes(k))
+    const used = entries.map(() => false)
 
-    // First pass: replay surviving keyOrder keys
+    // First pass: replay surviving keyOrder keys. Each keyOrder key takes the
+    // first equal entry that no earlier keyOrder key took, so a repeated key
+    // keeps the value it had at that position
     for (let j = 0; j < mapFmt.keyOrder.length; j++) {
       const key = internalDecodeSync(mapFmt.keyOrder[j])
-      decodedKeyOrderKeys.push(key)
-      const mapped = mapGetCBOR(value, key)
-      if (mapped !== undefined) {
-        pairs.push([key, mapped])
+      let i = plainKeys.findIndex((k, n) => !used[n] && equals(key, k))
+      if (i === -1) i = plainKeys.findIndex((k) => equals(key, k))
+      if (i !== -1) {
+        used[i] = true
+        pairs.push([key, entries[i][1]])
         reorderedEntries.push(mapFmt.entries[j] ?? [{ _tag: "simple" }, { _tag: "simple" }])
       }
       // Key missing from map: simply skip (key was removed)
     }
 
     // Second pass: append new keys not covered by keyOrder
-    for (const [key, val] of value) {
-      if (!decodedKeyOrderKeys.some((k) => equals(key, k))) {
-        pairs.push([key, val])
+    for (let n = 0; n < entries.length; n++) {
+      if (!used[n]) {
+        pairs.push(entries[n])
         reorderedEntries.push([{ _tag: "simple" }, { _tag: "simple" }])
       }
     }

packages/evolution/test/Transaction.DataMapBytesKey.test.ts

@@ -0,0 +1,84 @@
+import { Schema } from "effect"
+import { describe, expect, it } from "vitest"
+
+import * as Bytes from "../src/Bytes.js"
+import * as CBOR from "../src/CBOR.js"
+import * as Data from "../src/Data.js"
+import * as Transaction from "../src/Transaction.js"
+
+// A Plutus data map with byte-string keys inside a witness datum or redeemer
+// must keep its original bytes after decode and re-encode, because the script
+// data hash covers those bytes. Integer keys always kept them; byte keys lost
+// the format of the value under each key.
+
+// Real transaction; only its body is used, with hand-built witness sets
+const FIXTURE_TX_HEX =
+  "84ab00d90102818258208078d91b024d72d18a00125c0da1c5bc9405060a2cefbeadbade8133607f57eb010182a3005839104a965ad53909a5fe9f4c931c91c286f767c306647ea35ba326d3ef9e2decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8c01821a001bd19aa1581c87cb436d92d733c239120af93f1988bd49709eecdbe6b93216992fcaa24341736b0158206c6f616e836cc68931c2e4e3e838602eca1902591d216837bafddfe6f0c8cb0701028201d8185874d8799fd8799f581c87cb436d92d733c239120af93f1988bd49709eecdbe6b93216992fca581c34c89038e26522b4e4c627d71620cec18edae05a7ae94ee90e0e76b6581d002decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8c9f4040ff187b1aa4cb80009f9f4040ffffffff82583900e8a20635288ddfe3c916daa6fc7ce150136d83b30cc87c860a020b342decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8c821a6c02024ba8581c1855a70da3f8b041ff49a6ca063817598b1f7c72d4ef25a292e776f2a1560014df104578707265737320506f6c6c696e61746f7201581c34c89038e26522b4e4c627d71620cec18edae05a7ae94ee90e0e76b6a158206b6696daa9fa0fe7fb3136611113f5f64943a7b6b5a2a50d707ace02ddb561e301581c5ea37393bca4c996086a5d603a748c655f6f8a8c4be742fd1e0f1f64a1447457495a1a0bebc200581c985bb5ad24f633730635cc6501ed5ae8c8e490b6435c67ae58c36665a7480014df105553444d1b000002ba6d437267480014df10574d54581b000002ba7f2f9f40480014df10744941471b000002bcd1708a82490014df1074534e454b1a04c4ca064a0014df1074484f534b591a049d5d514b0014df107453554e4441451b000002bb9890fd084e0014df1057697a617264546573741a00989680581ca9fc2c980e6beed499b91089ca06ad433961a6238690219b8021fe43a3480014df1048554e541b000000037e11d600480014df10494e44591a0436b2494a0014df1053554e4441451b00000005d1ff3576581cafc338a86087b2389ff6e4e14d3305c6994c9908e6b50c6081916390a1447457495a1a05054ec1581cc08487509756b73dd0bd8406454bf0c83da952ca44454941a116d239a14574534e454b1a0a4d3b56581cd0d763bd8d1d6456989405ede72ca5c967351bbad2f2319daf54f9dca144744941471a089f2103021a00062f0d031a072752e309a1581c87cb436d92d733c239120af93f1988bd49709eecdbe6b93216992fcaa24341736b0158206c6f616e836cc68931c2e4e3e838602eca1902591d216837bafddfe6f0c8cb07010b5820d6e618039f9269fc9b58c4fefc09f277dbddfab32c76f340fa06786c48e6be9b0dd90102818258208d3894c757da4206026eaf45ee1c7d5bd8b38d7dd88f170d9aaf785fc7c52bc5000ed9010281581c2decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8c1082583900e8a20635288ddfe3c916daa6fc7ce150136d83b30cc87c860a020b342decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8c1a00346119111a0017ea2712d9010281825820b83d379cb8ef97f5514c5a9b6a18833c01d244ea9cff7e6a9a84c7a546f3fe6700a105a182010082d8799fd8799f581c2decd801bc24246797eb8c35bbdf98ed854d4a08a806853006817e8cffff821a000319db1a03b9dee0f5f6"
+
+const BODY_HEX = FIXTURE_TX_HEX.slice(2, CBOR.decodeItemWithOffset(Bytes.fromHex(FIXTURE_TX_HEX), 1).newOffset * 2)
+
+const EX_UNITS_HEX = "821a000319db1a03b9dee0"
+
+const WALLET_WITNESS_SET_HEX = "a10081825820" + "aa".repeat(32) + "5840" + "bb".repeat(64)
+
+// 70-byte key, written as a chunked byte string (64 + 6) as Plutus data requires
+const LONG_KEY_HEX = "5f5840" + "ab".repeat(64) + "46" + "cd".repeat(6) + "ff"
+
+const DATA_CASES: ReadonlyArray<readonly [string, string]> = [
+  ["definite map, byte key, indefinite list value", "d8799fa141019f0102ffff"],
+  ["indefinite map, byte key, indefinite list value", "d8799fbf41019f0102ffffff"],
+  ["chunked byte key longer than 64 bytes", "d8799fa1" + LONG_KEY_HEX + "9f0102ffff"],
+  ["two byte keys in unsorted order", "d8799fa241029f01ff41019f02ffff"],
+  ["list key holding a byte string", "d8799fa18141019f0102ffff"],
+  ["constr key holding a byte string", "d8799fa1d8798141019f0102ffff"],
+  ["repeated byte key with different values", "d8799fa241019f01ff41019f02ffff"],
+  ["integer key (control)", "d8799fa1019f0102ffff"],
+  ["all definite (control)", "d87981a14101820102"]
+]
+
+const datumWitnessSet = (data: string) => "a104d9010281" + data
+const redeemerArrayWitnessSet = (data: string) => "a10581840000" + data + EX_UNITS_HEX
+const redeemerMapWitnessSet = (data: string) => "a105a1820000" + "82" + data + EX_UNITS_HEX
+
+const WITNESS_FORMS = [
+  ["witness datum", datumWitnessSet],
+  ["redeemer array form", redeemerArrayWitnessSet],
+  ["redeemer map form", redeemerMapWitnessSet]
+] as const
+
+const txHex = (witnessSetHex: string) => "84" + BODY_HEX + witnessSetHex + "f5f6"
+
+describe("Transaction keeps Plutus data maps with byte-string keys", () => {
+  for (const [form, witnessSet] of WITNESS_FORMS) {
+    describe(form, () => {
+      for (const [name, data] of DATA_CASES) {
+        it(`fromCBORHex then toCBORHex round-trips: ${name}`, () => {
+          const hex = txHex(witnessSet(data))
+          expect(Transaction.toCBORHex(Transaction.fromCBORHex(hex))).toBe(hex)
+        })
+
+        it(`addVKeyWitnessesHex keeps the original witness bytes: ${name}`, () => {
+          const witnessSetHex = witnessSet(data)
+          const signed = Transaction.addVKeyWitnessesHex(txHex(witnessSetHex), WALLET_WITNESS_SET_HEX)
+
+          expect(signed.startsWith("84" + BODY_HEX)).toBe(true)
+          // The original entry is replayed first, with the vkey witnesses appended after it
+          expect(signed.slice(2 + BODY_HEX.length, 4 + BODY_HEX.length)).toBe("a2")
+          expect(signed.slice(4 + BODY_HEX.length)).toContain(witnessSetHex.slice(2))
+          expect(Transaction.fromCBORHex(signed).witnessSet.vkeyWitnesses?.length).toBe(1)
+        })
+      }
+    })
+  }
+})
+
+describe("CBOR.toCBORHexWithFormat on a Data map with byte-string keys", () => {
+  for (const [name, hex] of DATA_CASES) {
+    it(`reproduces the original bytes: ${name}`, () => {
+      const { format } = CBOR.fromCBORHexWithFormat(hex)
+      const data = Data.fromCBORHex(hex)
+      const cbor = Schema.encodeSync(Data.FromCDDL)(data)
+      expect(CBOR.toCBORHexWithFormat(cbor, format)).toBe(hex)
+    })
+  }
+})