fix(datum): keep the original bytes of decoded inline datums

.changeset/inline-datum-bytes.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+Decoding then re-encoding a transaction now keeps each inline datum's original bytes, so adding witnesses to a decoded transaction keeps its transaction id. Previously the encoder rewrote an inline datum whose layout differed from the default, for example turning a definite map `a10102` into `bf0102ff`.

packages/evolution-devnet/test/Transaction.InlineDatumBytes.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Devnet test: decoding a transaction and adding a witness keeps every inline
+ * datum's original bytes.
+ *
+ * The node keeps each inline datum exactly as written and the transaction id is
+ * the hash of the body bytes. A transaction carrying one inline datum as a
+ * definite map (a1...) and one as an indefinite map (bf...) is signed over its
+ * raw body, merged with Transaction.addVKeyWitnessesBytes, submitted as raw
+ * bytes, and the datums are read back from the ledger.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "@effect/vitest"
+import * as Cluster from "@evolution-sdk/devnet/Cluster"
+import * as Config from "@evolution-sdk/devnet/Config"
+import * as Genesis from "@evolution-sdk/devnet/Genesis"
+import { Cardano, Client, preprod } from "@evolution-sdk/evolution"
+import * as Address from "@evolution-sdk/evolution/Address"
+import * as Bytes from "@evolution-sdk/evolution/Bytes"
+import * as PlutusData from "@evolution-sdk/evolution/Data"
+import * as InlineDatum from "@evolution-sdk/evolution/InlineDatum"
+import * as Transaction from "@evolution-sdk/evolution/Transaction"
+import * as TransactionBody from "@evolution-sdk/evolution/TransactionBody"
+import * as TransactionHash from "@evolution-sdk/evolution/TransactionHash"
+import * as TransactionWitnessSet from "@evolution-sdk/evolution/TransactionWitnessSet"
+
+describe("Inline datum bytes (Devnet Submit)", () => {
+  let devnetCluster: Cluster.Cluster | undefined
+  let genesisUtxos: ReadonlyArray<Cardano.UTxO.UTxO> = []
+
+  const TEST_MNEMONIC =
+    "test test test test test test test test test test test test test test test test test test test test test test test sauce"
+
+  const ogmiosUrl = () => `http://localhost:${devnetCluster!.ports.ogmios}`
+
+  const createTestClient = () => {
+    if (!devnetCluster) throw new Error("Cluster not initialized")
+    return Client.make(Cluster.getChain(devnetCluster))
+      .withKupmios({ kupoUrl: `http://localhost:${devnetCluster.ports.kupo}`, ogmiosUrl: ogmiosUrl() })
+      .withSeed({ mnemonic: TEST_MNEMONIC, accountIndex: 0, addressType: "Base" })
+  }
+
+  const ogmios = async (method: string, params: unknown) => {
+    const response = await fetch(ogmiosUrl(), {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ jsonrpc: "2.0", method, params, id: null })
+    })
+    return (await response.json()) as { result?: any; error?: unknown }
+  }
+
+  beforeAll(async () => {
+    const tempClient = Client.make(preprod).withSeed({ mnemonic: TEST_MNEMONIC, accountIndex: 0, addressType: "Base" })
+    const testAddressHex = Address.toHex(await tempClient.address())
+
+    const genesisConfig: Config.ShelleyGenesis = {
+      ...Config.DEFAULT_SHELLEY_GENESIS,
+      slotLength: 0.02,
+      epochLength: 50,
+      activeSlotsCoeff: 1.0,
+      initialFunds: { [testAddressHex]: 500_000_000_000 }
+    }
+
+    genesisUtxos = await Genesis.calculateUtxosFromConfig(genesisConfig)
+
+    devnetCluster = await Cluster.make({
+      clusterName: "inline-datum-bytes-test",
+      shelleyGenesis: genesisConfig,
+      kupo: { enabled: true, logLevel: "Info" },
+      ogmios: { enabled: true, logLevel: "info" }
+    })
+
+    await Cluster.start(devnetCluster)
+    await new Promise((resolve) => setTimeout(resolve, 3_000))
+  }, 180_000)
+
+  afterAll(async () => {
+    if (devnetCluster) {
+      await Cluster.stop(devnetCluster)
+      await Cluster.remove(devnetCluster)
+    }
+  }, 60_000)
+
+  it("keeps definite and indefinite map datums through addVKeyWitnessesBytes", { timeout: 90_000 }, async () => {
+    const client = createTestClient()
+    const myAddress = await client.address()
+
+    // The builder writes both maps indefinite (bf...ff). The second one is then
+    // rewritten as a definite map (a1...), one byte shorter, so the fee still covers it.
+    const indefiniteDatum = "bf0102ff"
+    const definiteDatum = "a10304"
+    const signBuilder = await client
+      .newTx()
+      .payToAddress({
+        address: myAddress,
+        assets: Cardano.Assets.fromLovelace(5_000_000n),
+        datum: new InlineDatum.InlineDatum({ data: PlutusData.map([[PlutusData.int(1n), PlutusData.int(2n)]]) })
+      })
+      .payToAddress({
+        address: myAddress,
+        assets: Cardano.Assets.fromLovelace(5_000_000n),
+        datum: new InlineDatum.InlineDatum({ data: PlutusData.map([[PlutusData.int(3n), PlutusData.int(4n)]]) })
+      })
+      .build({ availableUtxos: [...genesisUtxos] })
+
+    const builtHex = Transaction.toCBORHex(await signBuilder.toTransaction())
+    expect(builtHex.split("d81844" + indefiniteDatum).length).toBe(2)
+    expect(builtHex.split("d81844bf0304ff").length).toBe(2)
+    const txHex = builtHex.replace("d81844bf0304ff", "d81843" + definiteDatum)
+    const txBytes = Bytes.fromHex(txHex)
+    const bodyBytes = Transaction.extractBodyBytes(txBytes)
+    const expectedTxId = TransactionHash.toHex(TransactionBody.toHashFromBytes(bodyBytes))
+
+    // Sign over the raw body, then merge the witness into the decoded transaction
+    const walletWitnessSet = await client.signTx(txHex, { utxos: [...genesisUtxos] })
+    expect(walletWitnessSet.vkeyWitnesses?.length).toBe(1)
+    const signedBytes = Transaction.addVKeyWitnessesBytes(txBytes, TransactionWitnessSet.toCBORBytes(walletWitnessSet))
+
+    const submitted = await ogmios("submitTransaction", { transaction: { cbor: Bytes.toHex(signedBytes) } })
+    expect(submitted.error).toBeUndefined()
+    expect(submitted.result.transaction.id).toBe(expectedTxId)
+    expect(Bytes.toHex(Transaction.extractBodyBytes(signedBytes))).toBe(Bytes.toHex(bodyBytes))
+
+    expect(await client.awaitTx(TransactionHash.fromHex(expectedTxId), 1000)).toBe(true)
+
+    const utxos = await ogmios("queryLedgerState/utxo", {
+      outputReferences: [
+        { transaction: { id: expectedTxId }, index: 0 },
+        { transaction: { id: expectedTxId }, index: 1 }
+      ]
+    })
+    const datums = (utxos.result as Array<{ index: number; datum?: string }>)
+      .sort((a, b) => a.index - b.index)
+      .map((utxo) => utxo.datum)
+    expect(datums).toEqual([indefiniteDatum, definiteDatum])
+  })
+})

packages/evolution/src/DatumOption.ts

@@ -4,6 +4,7 @@ import * as CBOR from "./CBOR.js"
 import * as PlutusData from "./Data.js"
 import * as DatumHash from "./DatumHash.js"
 import * as InlineDatum from "./InlineDatum.js"
+import * as OriginalBytes from "./OriginalBytes.js"
 
 /**
  * Schema for DatumOption representing optional datum information in transaction outputs.
@@ -77,7 +78,15 @@ export const FromCDDL = Schema.transformOrFail(CDDLSchema, Schema.typeSchema(Dat
       const result =
         toA._tag === "DatumHash"
           ? ([0n, toA.hash] as const) // Encode as [0, Bytes32]
-          : ([1n, { _tag: "Tag" as const, tag: 24 as const, value: PlutusData.toCBORBytes(toA.data) }] as const) // Encode as [1, tag(24, bytes)]
+          : ([
+              1n,
+              {
+                _tag: "Tag" as const,
+                tag: 24 as const,
+                // Write back the decoded bytes when present, so the datum keeps its original layout
+                value: OriginalBytes.get(toA) ?? PlutusData.toCBORBytes(toA.data)
+              }
+            ] as const) // Encode as [1, tag(24, bytes)]
       return yield* E.right(result)
     }),
   decode: ([tag, value], _, ast) =>
@@ -98,11 +107,14 @@ export const FromCDDL = Schema.transformOrFail(CDDLSchema, Schema.typeSchema(Dat
           )
         }
         return yield* E.right(
-          new InlineDatum.InlineDatum(
-            {
-              data: PlutusData.fromCBORBytes(taggedValue.value)
-            },
-            { disableValidation: true }
+          OriginalBytes.record(
+            new InlineDatum.InlineDatum(
+              {
+                data: PlutusData.fromCBORBytes(taggedValue.value)
+              },
+              { disableValidation: true }
+            ),
+            taggedValue.value
           )
         )
       }

packages/evolution/src/OriginalBytes.ts

@@ -0,0 +1,35 @@
+/**
+ * Keeps the exact bytes a value was decoded from.
+ *
+ * The ledger hashes and signs the original bytes of a value, not a fresh
+ * encoding of it. A decoder records the bytes it read against the object it
+ * returns. An encoder that finds recorded bytes for the object it is given
+ * writes those bytes back unchanged. A new or edited value is a new object,
+ * so it has no recorded bytes and gets the default encoding.
+ *
+ * Entries are held in a WeakMap, so they are released with their object.
+ *
+ * @since 2.0.0
+ */
+
+const cache = new WeakMap<object, Uint8Array>()
+
+/**
+ * Record the bytes that `value` was decoded from. The bytes are copied, so a
+ * caller that later changes its input buffer does not change the record.
+ *
+ * @since 2.0.0
+ * @category cache
+ */
+export const record = <A extends object>(value: A, bytes: Uint8Array): A => {
+  cache.set(value, bytes.slice())
+  return value
+}
+
+/**
+ * Get the bytes that `value` was decoded from, if any were recorded.
+ *
+ * @since 2.0.0
+ * @category cache
+ */
+export const get = (value: object): Uint8Array | undefined => cache.get(value)

packages/evolution/src/index.ts

@@ -76,6 +76,7 @@ export * as NonnegativeInterval from "./NonnegativeInterval.js"
 export * as NonZeroInt64 from "./NonZeroInt64.js"
 export * as Numeric from "./Numeric.js"
 export * as OperationalCert from "./OperationalCert.js"
+export * as OriginalBytes from "./OriginalBytes.js"
 export * as PaymentAddress from "./PaymentAddress.js"
 export * as PlutusV1 from "./PlutusV1.js"
 export * as PlutusV2 from "./PlutusV2.js"

packages/evolution/test/DatumOption.OriginalBytes.test.ts

@@ -0,0 +1,121 @@
+import { blake2b } from "@noble/hashes/blake2.js"
+import { Schema } from "effect"
+import { describe, expect, it } from "vitest"
+
+import * as PlutusData from "../src/Data.js"
+import * as DatumOption from "../src/DatumOption.js"
+import * as InlineDatum from "../src/InlineDatum.js"
+import * as PrivateKey from "../src/PrivateKey.js"
+import * as Transaction from "../src/Transaction.js"
+import * as TransactionOutput from "../src/TransactionOutput.js"
+import * as TxOut from "../src/TxOut.js"
+import * as VKey from "../src/VKey.js"
+
+// The ledger keeps the original bytes of every inline datum (MemoBytes), and the
+// transaction id is the hash of the body bytes. Decoding a transaction and
+// encoding it again must write each inline datum back byte for byte.
+
+const hex = (bytes: Uint8Array) => Buffer.from(bytes).toString("hex")
+const fromHex = (h: string) => new Uint8Array(Buffer.from(h, "hex"))
+
+/** CBOR byte string header for a length below 65536. */
+const bstr = (h: string) => {
+  const n = h.length / 2
+  if (n < 24) return (0x40 + n).toString(16).padStart(2, "0") + h
+  if (n < 256) return "58" + n.toString(16).padStart(2, "0") + h
+  return "59" + n.toString(16).padStart(4, "0") + h
+}
+
+const sixtyFour = "11".repeat(64)
+
+const cases: ReadonlyArray<{ readonly name: string; readonly datum: string }> = [
+  { name: "constructor holding a definite map", datum: "d8799fa10102ff" },
+  { name: "bare definite map", datum: "a10102" },
+  { name: "indefinite map", datum: "bf0102ff" },
+  { name: "constructor with indefinite fields", datum: "d8799f0102ff" },
+  { name: "constructor with definite fields", datum: "d879820102" },
+  { name: "bare integer", datum: "05" },
+  { name: "65 bytes chunked 64 then 1", datum: "5f" + bstr(sixtyFour) + "4122ff" },
+  { name: "65 bytes chunked 1 then 64", datum: "5f4122" + bstr(sixtyFour) + "ff" }
+]
+
+const txId = "aa".repeat(32)
+const address = "60" + "bb".repeat(28)
+
+/** Body {0: 258([[txid, 0]]), 1: [{0: addr, 1: 1000000, 2: [1, 24(datum)]}], 2: 200000}. */
+const bodyHex = (datum: string) =>
+  "a3" +
+  "00d901028182" + bstr(txId) + "00" +
+  "0181a300" + bstr(address) + "011a000f4240" + "028201d818" + bstr(datum) +
+  "021a00030d40"
+
+/** Transaction [body, {}, true, null]. */
+const txHex = (datum: string) => "84" + bodyHex(datum) + "a0f5f6"
+
+const bodyOf = (txHexValue: string) => hex(Transaction.extractBodyBytes(fromHex(txHexValue)))
+const hashOf = (body: string) => hex(blake2b(fromHex(body), { dkLen: 32 }))
+
+/** Witness set {0: [[vkey, signature]]} signed over the body hash. */
+const witnessSetHex = (body: string) => {
+  const key = PrivateKey.fromBytes(new Uint8Array(32).fill(7))
+  const vkey = VKey.toBytes(PrivateKey.toPublicKey(key))
+  const signature = PrivateKey.sign(key, blake2b(fromHex(body), { dkLen: 32 }))
+  return "a1008182" + bstr(hex(vkey)) + bstr(hex(signature.bytes))
+}
+
+describe("inline datum keeps its original bytes", () => {
+  for (const { datum, name } of cases) {
+    describe(`${name} (${datum.length > 40 ? datum.slice(0, 40) + "..." : datum})`, () => {
+      const tx = txHex(datum)
+      const body = bodyHex(datum)
+
+      it("the hand-built transaction carries the datum as given", () => {
+        expect(bodyOf(tx)).toBe(body)
+      })
+
+      it("Transaction.fromCBORHex then toCBORHex keeps the body and its hash", () => {
+        const out = Transaction.toCBORHex(Transaction.fromCBORHex(tx))
+        expect(bodyOf(out)).toBe(body)
+        expect(hashOf(bodyOf(out))).toBe(hashOf(body))
+      })
+
+      it("the WithFormat round trip keeps the body", () => {
+        const { format, value } = Transaction.fromCBORHexWithFormat(tx)
+        expect(bodyOf(Transaction.toCBORHexWithFormat(value, format))).toBe(body)
+      })
+
+      it("the Schema path FromCBORHex keeps the body", () => {
+        const schema = Transaction.FromCBORHex()
+        const out = Schema.encodeSync(schema)(Schema.decodeSync(schema)(tx))
+        expect(bodyOf(out)).toBe(body)
+      })
+
+      it("addVKeyWitnessesHex keeps the body hash", () => {
+        const signed = Transaction.addVKeyWitnessesHex(tx, witnessSetHex(body))
+        expect(signed).not.toBe(tx)
+        expect(hashOf(bodyOf(signed))).toBe(hashOf(body))
+      })
+
+      it("TransactionOutput and TxOut round trips keep the datum", () => {
+        const output = "a300" + bstr(address) + "011a000f4240" + "028201d818" + bstr(datum)
+        expect(TransactionOutput.toCBORHex(TransactionOutput.fromCBORHex(output))).toBe(output)
+        expect(TxOut.toCBORHex(TxOut.fromCBORHex(output))).toBe(output)
+      })
+
+      it("DatumOption round trip keeps the datum", () => {
+        const option = "8201d818" + bstr(datum)
+        expect(DatumOption.toCBORHex(DatumOption.fromCBORHex(option))).toBe(option)
+      })
+    })
+  }
+
+  it("a new InlineDatum with the same Data uses the default encoding", () => {
+    const option = "8201d818" + bstr("d8799fa10102ff")
+    const decoded = DatumOption.fromCBORHex(option)
+    if (!InlineDatum.isInlineDatum(decoded)) throw new Error("expected an inline datum")
+    const edited = new InlineDatum.InlineDatum({ data: decoded.data })
+    const expected = "8201d818" + bstr(PlutusData.toCBORHex(decoded.data))
+    expect(expected).toBe("8201d818" + bstr("d8799fbf0102ffff"))
+    expect(DatumOption.toCBORHex(edited)).toBe(expected)
+  })
+})