re-add XXH64 to read borg 1.x integrity data, #9935

src/borg/crypto/file_integrity.py

@@ -5,6 +5,7 @@
 from collections.abc import Callable
 from pathlib import Path
 
+from .low_level import XXH64
 from ..helpers import IntegrityError
 from ..logger import create_logger
 
@@ -110,7 +111,19 @@ class SHA256FileHashingWrapper(FileHashingWrapper):
     FACTORY = hashlib.sha256
 
 
-SUPPORTED_ALGORITHMS = {SHA256FileHashingWrapper.ALGORITHM: SHA256FileHashingWrapper}
+class XXH64FileHashingWrapper(FileHashingWrapper):
+    # This only exists to support `borg transfer` from borg 1.x repos, see #9935:
+    # borg 1.x wrote XXH64 integrity data for the repo index/hints files, so we need
+    # XXH64 to verify those when reading a borg 1.x (legacy) repository. borg 2.x writes
+    # SHA256 (see SHA256FileHashingWrapper), so XXH64 is only ever used for reading here.
+    ALGORITHM = "XXH64"
+    FACTORY = XXH64
+
+
+SUPPORTED_ALGORITHMS = {
+    SHA256FileHashingWrapper.ALGORITHM: SHA256FileHashingWrapper,
+    XXH64FileHashingWrapper.ALGORITHM: XXH64FileHashingWrapper,
+}
 
 
 class FileIntegrityError(IntegrityError):

src/borg/crypto/low_level.pyi

@@ -13,6 +13,20 @@ def hmac_sha256(key: bytes, data: bytes) -> bytes: ...
 def blake2b_256(key: bytes, data: bytes) -> bytes: ...
 def blake2b_128(data: bytes) -> bytes: ...
 
+# XXH64: only used to read borg 1.x repo integrity data on `borg transfer`, see #9935.
+def xxh64(data: bytes, seed: int = 0) -> bytes: ...
+
+class XXH64:
+    """Streaming XXH64 hasher (non-cryptographic), compatible with the "xxhash" package's xxh64.
+
+    Only exists to read borg 1.x repo integrity data on `borg transfer`, see #9935.
+    """
+
+    def __init__(self, data: bytes = b"", seed: int = 0) -> None: ...
+    def update(self, data: bytes) -> None: ...
+    def digest(self) -> bytes: ...
+    def hexdigest(self) -> str: ...
+
 # Exception classes
 class CryptoError(Exception):
     """Malfunction in the crypto module."""

src/borg/crypto/low_level.pyx

@@ -825,3 +825,174 @@ cdef class CSPRNG:
 
             # Swap items[i] and items[j]
             items[i], items[j] = items[j], items[i]
+
+
+# XXH64: a pure-cython, dependency-free implementation of the (non-cryptographic) xxHash-64 hash.
+#
+# This only exists to support `borg transfer` from borg 1.x repos, see #9935:
+# borg 1.x stored XXH64 checksums in the repository's index/hints integrity data, so we need
+# XXH64 to verify those files when opening a borg 1.x (legacy) repository. It is not used for
+# anything in borg 2.x native repos and MUST NOT be used as a security mechanism.
+#
+# Reference: https://github.com/Cyan4973/xxHash (algorithm is in the public domain / BSD-2-Clause).
+# The digest uses the canonical (big-endian) representation, matching what borg 1.x wrote
+# (it used the "xxhash" PyPI package, whose digest()/hexdigest() are big-endian).
+
+cdef extern from *:
+    """
+    #include <stdint.h>
+    /* xxHash-64 prime constants, defined as real uint64_t so there is no ambiguity
+       about the width/signedness of these (large) literals. */
+    static const uint64_t BORG_XXH64_P1 = 0x9E3779B185EBCA87ULL;
+    static const uint64_t BORG_XXH64_P2 = 0xC2B2AE3D27D4EB4FULL;
+    static const uint64_t BORG_XXH64_P3 = 0x165667B19E3779F9ULL;
+    static const uint64_t BORG_XXH64_P4 = 0x85EBCA77C2B2AE63ULL;
+    static const uint64_t BORG_XXH64_P5 = 0x27D4EB2F165667C5ULL;
+    """
+    const uint64_t BORG_XXH64_P1
+    const uint64_t BORG_XXH64_P2
+    const uint64_t BORG_XXH64_P3
+    const uint64_t BORG_XXH64_P4
+    const uint64_t BORG_XXH64_P5
+
+
+cdef inline uint64_t _xxh_rotl(uint64_t x, int r) noexcept:
+    return (x << r) | (x >> (64 - r))
+
+
+cdef inline uint64_t _xxh_round(uint64_t acc, uint64_t inp) noexcept:
+    acc += inp * BORG_XXH64_P2
+    acc = _xxh_rotl(acc, 31)
+    acc *= BORG_XXH64_P1
+    return acc
+
+
+cdef inline uint64_t _xxh_merge(uint64_t acc, uint64_t val) noexcept:
+    acc ^= _xxh_round(0, val)
+    acc = acc * BORG_XXH64_P1 + BORG_XXH64_P4
+    return acc
+
+
+# read 64/32 bits little-endian, byte-wise, so this is correct on both little- and big-endian hosts.
+cdef inline uint64_t _xxh_read64(const uint8_t *p) noexcept:
+    return (<uint64_t>p[0] | (<uint64_t>p[1] << 8) | (<uint64_t>p[2] << 16) | (<uint64_t>p[3] << 24) |
+            (<uint64_t>p[4] << 32) | (<uint64_t>p[5] << 40) | (<uint64_t>p[6] << 48) | (<uint64_t>p[7] << 56))
+
+
+cdef inline uint32_t _xxh_read32(const uint8_t *p) noexcept:
+    return (<uint32_t>p[0] | (<uint32_t>p[1] << 8) | (<uint32_t>p[2] << 16) | (<uint32_t>p[3] << 24))
+
+
+cdef class XXH64:
+    """
+    Streaming XXH64 hasher with an interface compatible with the "xxhash" PyPI package's xxh64
+    (as used by borg 1.x): XXH64([data], [seed]) then .update(data) and .digest()/.hexdigest().
+
+    See the comment above: this exists only to support `borg transfer` from borg 1.x repos, #9935.
+    """
+    cdef uint64_t v1, v2, v3, v4
+    cdef uint64_t total_len
+    cdef uint64_t seed
+    cdef uint8_t mem[32]
+    cdef unsigned int memsize
+
+    def __init__(self, data=b"", seed=0):
+        # coerce seed into a C uint64_t first, so the setup arithmetic below wraps in C
+        # (mod 2**64) instead of overflowing as an unbounded python int.
+        cdef uint64_t s = seed
+        self.seed = s
+        self.v1 = s + BORG_XXH64_P1 + BORG_XXH64_P2
+        self.v2 = s + BORG_XXH64_P2
+        self.v3 = s
+        self.v4 = s - BORG_XXH64_P1
+        self.total_len = 0
+        self.memsize = 0
+        if data:
+            self.update(data)
+
+    def update(self, data):
+        cdef Py_buffer view
+        PyObject_GetBuffer(data, &view, PyBUF_SIMPLE)
+        try:
+            self._update(<const uint8_t *> view.buf, view.len)
+        finally:
+            PyBuffer_Release(&view)
+
+    cdef void _update(self, const uint8_t *p, Py_ssize_t length) noexcept:
+        cdef const uint8_t *end = p + length
+        cdef const uint8_t *limit
+        cdef unsigned int fill
+        self.total_len += length
+        if self.memsize + length < 32:
+            # not enough (even together with buffered data) for a full 32-byte stripe: just buffer it.
+            memcpy(&self.mem[self.memsize], p, length)
+            self.memsize += <unsigned int> length
+            return
+        if self.memsize > 0:
+            # complete and process the buffered partial stripe first.
+            fill = 32 - self.memsize
+            memcpy(&self.mem[self.memsize], p, fill)
+            self.v1 = _xxh_round(self.v1, _xxh_read64(&self.mem[0]))
+            self.v2 = _xxh_round(self.v2, _xxh_read64(&self.mem[8]))
+            self.v3 = _xxh_round(self.v3, _xxh_read64(&self.mem[16]))
+            self.v4 = _xxh_round(self.v4, _xxh_read64(&self.mem[24]))
+            p += fill
+            self.memsize = 0
+        # process full 32-byte stripes directly from the input.
+        limit = end - 32
+        while p <= limit:
+            self.v1 = _xxh_round(self.v1, _xxh_read64(p)); p += 8
+            self.v2 = _xxh_round(self.v2, _xxh_read64(p)); p += 8
+            self.v3 = _xxh_round(self.v3, _xxh_read64(p)); p += 8
+            self.v4 = _xxh_round(self.v4, _xxh_read64(p)); p += 8
+        # buffer the remaining (< 32) bytes for the next update()/digest().
+        if p < end:
+            memcpy(&self.mem[0], p, end - p)
+            self.memsize = <unsigned int> (end - p)
+
+    def digest(self):
+        """Return the digest as 8 bytes, in canonical (big-endian) representation."""
+        cdef uint64_t h
+        cdef const uint8_t *p = &self.mem[0]
+        cdef const uint8_t *end = &self.mem[self.memsize]
+        cdef uint8_t out[8]
+        cdef int i
+        if self.total_len >= 32:
+            h = _xxh_rotl(self.v1, 1) + _xxh_rotl(self.v2, 7) + _xxh_rotl(self.v3, 12) + _xxh_rotl(self.v4, 18)
+            h = _xxh_merge(h, self.v1)
+            h = _xxh_merge(h, self.v2)
+            h = _xxh_merge(h, self.v3)
+            h = _xxh_merge(h, self.v4)
+        else:
+            h = self.seed + BORG_XXH64_P5
+        h += self.total_len
+        while p + 8 <= end:
+            h ^= _xxh_round(0, _xxh_read64(p))
+            h = _xxh_rotl(h, 27) * BORG_XXH64_P1 + BORG_XXH64_P4
+            p += 8
+        if p + 4 <= end:
+            h ^= <uint64_t> _xxh_read32(p) * BORG_XXH64_P1
+            h = _xxh_rotl(h, 23) * BORG_XXH64_P2 + BORG_XXH64_P3
+            p += 4
+        while p < end:
+            h ^= <uint64_t> p[0] * BORG_XXH64_P5
+            h = _xxh_rotl(h, 11) * BORG_XXH64_P1
+            p += 1
+        # final avalanche
+        h ^= h >> 33
+        h *= BORG_XXH64_P2
+        h ^= h >> 29
+        h *= BORG_XXH64_P3
+        h ^= h >> 32
+        for i in range(8):
+            out[i] = <uint8_t> (h >> (56 - 8 * i))  # big-endian
+        return PyBytes_FromStringAndSize(<char *> out, 8)
+
+    def hexdigest(self):
+        """Return the digest as a 16-character hex string (canonical, big-endian)."""
+        return self.digest().hex()
+
+
+def xxh64(data, seed=0):
+    """One-shot XXH64: return the 8-byte canonical (big-endian) digest of *data*."""
+    return XXH64(data, seed).digest()

src/borg/testsuite/crypto/file_integrity_test.py

@@ -4,7 +4,8 @@
 import pytest
 
 from ...crypto.file_integrity import DetachedIntegrityCheckedFile, FileIntegrityError, IntegrityCheckedFile
-from ...crypto.file_integrity import SHA256FileHashingWrapper
+from ...crypto.file_integrity import SHA256FileHashingWrapper, XXH64FileHashingWrapper, SUPPORTED_ALGORITHMS
+from ...crypto.low_level import XXH64
 from ...platform import SyncFile
 
 
@@ -177,3 +178,64 @@ def test_impure_hash_write(self):
         # pure_hash=False appends the file length ("11" in this case) at exit
         expected_hash = hashlib.sha256(data + b"11").hexdigest()
         assert wrapper.hexdigest() == expected_hash
+
+
+class TestXXH64FileHashingWrapper:
+    # XXH64 support only exists to read borg 1.x repos during `borg transfer`, see #9935.
+    def test_registered(self):
+        assert SUPPORTED_ALGORITHMS["XXH64"] is XXH64FileHashingWrapper
+        assert XXH64FileHashingWrapper.ALGORITHM == "XXH64"
+
+    def test_pure_hash_write(self):
+        bio = io.BytesIO()
+        data = b"hello world"
+        with XXH64FileHashingWrapper(bio, write=True, pure_hash=True) as wrapper:
+            wrapper.write(data)
+            assert bio.getvalue() == data
+        assert wrapper.hexdigest() == XXH64(data).hexdigest()
+
+    def test_pure_hash_read(self):
+        data = b"hello world"
+        bio = io.BytesIO(data)
+        with XXH64FileHashingWrapper(bio, write=False, pure_hash=True) as wrapper:
+            assert wrapper.read() == data
+        assert wrapper.hexdigest() == XXH64(data).hexdigest()
+
+    def test_impure_hash_write(self):
+        bio = io.BytesIO()
+        data = b"hello world"
+        with XXH64FileHashingWrapper(bio, write=True, pure_hash=False) as wrapper:
+            wrapper.write(data)
+        # pure_hash=False appends the file length ("11" in this case) at exit
+        assert wrapper.hexdigest() == XXH64(data + b"11").hexdigest()
+
+
+class TestReadLegacyXXH64Integrity:
+    # simulate reading a borg 1.x file whose integrity data uses the XXH64 algorithm, #9935.
+    # borg 2.x writes SHA256, so to produce an XXH64 integrity file we make the writer use the
+    # XXH64 wrapper (as borg 1.x did), then read it back through the normal (SHA256-defaulting)
+    # reader, which must pick XXH64 from SUPPORTED_ALGORITHMS based on the stored algorithm.
+    def _write_xxh64_protected(self, tmpdir, payload, monkeypatch):
+        import borg.crypto.file_integrity as fi
+
+        monkeypatch.setattr(fi, "SHA256FileHashingWrapper", XXH64FileHashingWrapper)
+        path = str(tmpdir.join("file"))
+        with IntegrityCheckedFile(path, write=True) as fd:
+            fd.write(payload)
+        integrity_data = fd.integrity_data
+        monkeypatch.undo()
+        assert '"algorithm": "XXH64"' in integrity_data
+        return path, integrity_data
+
+    def test_verify_ok(self, tmpdir, monkeypatch):
+        path, integrity_data = self._write_xxh64_protected(tmpdir, b"borg 1.x payload", monkeypatch)
+        with IntegrityCheckedFile(path, write=False, integrity_data=integrity_data) as fd:
+            assert fd.read() == b"borg 1.x payload"
+
+    def test_verify_detects_corruption(self, tmpdir, monkeypatch):
+        path, integrity_data = self._write_xxh64_protected(tmpdir, b"borg 1.x payload", monkeypatch)
+        with open(path, "ab") as fd:
+            fd.write(b" tampered")
+        with pytest.raises(FileIntegrityError):
+            with IntegrityCheckedFile(path, write=False, integrity_data=integrity_data) as fd:
+                fd.read()