From fe08e85c512d4238d2e4fdb041b17257b4d6d241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=B9=B310304955?= Date: Fri, 7 Aug 2026 10:33:19 +0800 Subject: [PATCH 1/5] gpfdist supports reading LZO-compressed files. --- configure | 113 ++++ configure.ac | 17 + devops/deploy/docker/build/rocky10/Dockerfile | 1 + devops/deploy/docker/build/rocky8/Dockerfile | 1 + devops/deploy/docker/build/rocky9/Dockerfile | 1 + devops/sandbox/Dockerfile.RELEASE.rockylinux9 | 1 + devops/sandbox/Dockerfile.main.rockylinux9 | 1 + src/Makefile.global.in | 6 +- src/backend/utils/misc/fstream/gfile.c | 619 +++++++++++++++++- src/include/fstream/gfile.h | 16 +- src/include/pg_config.h.in | 48 +- 11 files changed, 782 insertions(+), 42 deletions(-) diff --git a/configure b/configure index d2ffd6b00bd..87e8756033f 100755 --- a/configure +++ b/configure @@ -721,6 +721,8 @@ with_gp_stats_collector with_diskquota with_libbz2 with_zstd +LZO_LIBS +with_lzo LZ4_LIBS LZ4_CFLAGS with_lz4 @@ -944,6 +946,7 @@ with_zlib with_lz4 with_zstd with_libbz2 +with_lzo with_diskquota with_gp_stats_collector with_rt @@ -1697,6 +1700,7 @@ Optional Packages: --with-zstd build with ZSTD support --without-libbz2 do not use bzip2 --with-zstd do not build with Zstandard + --with-lzo build with LZO support --with-diskquota build with diskquota extension --with-gp-stats-collector build with stats collector extension @@ -11574,6 +11578,115 @@ fi $as_echo "$with_zstd" >&6; } +# +# LZO +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZO support" >&5 +$as_echo_n "checking whether to build with LZO support... " >&6; } + + + +# Check whether --with-lzo was given. +if test "${with_lzo+set}" = set; then : + withval=$with_lzo; + case $withval in + yes) + +$as_echo "#define USE_LZO 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lzo option" "$LINENO" 5 + ;; + esac + +else + with_lzo=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lzo" >&5 +$as_echo "$with_lzo" >&6; } + + +if test "$with_lzo" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking lzo/lzo1x.h usability" >&5 +$as_echo_n "checking lzo/lzo1x.h usability... " >&6; } +if ${ac_cv_header_lzo_lzo1x_h+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_includes_default +#include +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_header_lzo_lzo1x_h=yes +else + ac_cv_header_lzo_lzo1x_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_lzo_lzo1x_h" >&5 +$as_echo "$ac_cv_header_lzo_lzo1x_h" >&6; } + + +if test "x$ac_cv_header_lzo_lzo1x_h" = x""yes; then : + : +else + as_fn_error $? "lzo/lzo1x.h header not found. Please install liblzo2-dev." "$LINENO" 5 +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for lzo1x_decompress_safe in -llzo2" >&5 +$as_echo_n "checking for lzo1x_decompress_safe in -llzo2... " >&6; } +if ${ac_cv_lib_lzo2_lzo1x_decompress_safe+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-llzo2 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Suppress warnings for systems that declare it in a header. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char lzo1x_decompress_safe (); +int +main () +{ +return lzo1x_decompress_safe (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_lzo2_lzo1x_decompress_safe=yes +else + ac_cv_lib_lzo2_lzo1x_decompress_safe=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzo2_lzo1x_decompress_safe" >&5 +$as_echo "$ac_cv_lib_lzo2_lzo1x_decompress_safe" >&6; } +if test "x$ac_cv_lib_lzo2_lzo1x_decompress_safe" = x""yes; then : + LZO_LIBS="-llzo2" +else + as_fn_error $? "liblzo2 library not found. Please install liblzo2-dev." "$LINENO" 5 +fi + +fi + + # # diskquota # diff --git a/configure.ac b/configure.ac index dd259c4f07f..571af089f32 100644 --- a/configure.ac +++ b/configure.ac @@ -1466,6 +1466,23 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# LZO +# +AC_MSG_CHECKING([whether to build with LZO support]) +AH_TEMPLATE([USE_LZO], [Define to 1 to build with LZO support. (--with-lzo)]) +PGAC_ARG_BOOL(with, lzo, no, [build with LZO support], + [AC_DEFINE([USE_LZO], 1)]) +AC_MSG_RESULT([$with_lzo]) +AC_SUBST(with_lzo) + +if test "$with_lzo" = yes; then + AC_CHECK_HEADER([lzo/lzo1x.h], [], [AC_MSG_ERROR([lzo/lzo1x.h header not found. Please install liblzo2-dev.])]) + AC_CHECK_LIB([lzo2], [lzo1x_decompress_safe], [LZO_LIBS="-llzo2"], + [AC_MSG_ERROR([liblzo2 library not found. Please install liblzo2-dev.])]) + AC_SUBST(LZO_LIBS) +fi + # # diskquota # diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile index 3ab40229020..fc2720a2c08 100644 --- a/devops/deploy/docker/build/rocky10/Dockerfile +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -122,6 +122,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index eed55e4b171..a1f6eacb4e9 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -104,6 +104,7 @@ RUN dnf makecache && \ libuuid-devel \ libxml2-devel \ libzstd-devel \ + lzo-devel \ lsof \ lz4 \ lz4-devel \ diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 592d9b2e10a..17ff9601efc 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -124,6 +124,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 index 215c32f452d..7750a79b240 100644 --- a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 +++ b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 @@ -81,6 +81,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/devops/sandbox/Dockerfile.main.rockylinux9 b/devops/sandbox/Dockerfile.main.rockylinux9 index 03619cda4ba..676e230e6ca 100644 --- a/devops/sandbox/Dockerfile.main.rockylinux9 +++ b/devops/sandbox/Dockerfile.main.rockylinux9 @@ -86,6 +86,7 @@ RUN dnf makecache && \ libxml2-devel \ libuuid-devel \ libzstd-devel \ + lzo-devel \ lz4 \ lz4-devel \ openldap-devel \ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 74ba8d0c370..bd47d02c289 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -268,6 +268,8 @@ YAML_LIBS = @YAML_LIBS@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ +with_lzo = @with_lzo@ +LZO_LIBS = @LZO_LIBS@ EVENT_LIBS = @EVENT_LIBS@ with_diskquota = @with_diskquota@ with_gp_stats_collector = @with_gp_stats_collector@ @@ -335,8 +337,8 @@ LDAP_LIBS_BE = @LDAP_LIBS_BE@ UUID_LIBS = @UUID_LIBS@ LLVM_LIBS=@LLVM_LIBS@ -# Cloudberry uses zstd in frontend and backend -LIBS := $(LIBS) $(ZSTD_LIBS) +# Cloudberry uses zstd and lzo in frontend and backend +LIBS := $(LIBS) $(ZSTD_LIBS) $(LZO_LIBS) # It's critical that within LDFLAGS, all -L switches pointing to build-tree # directories come before any -L switches pointing to external directories. diff --git a/src/backend/utils/misc/fstream/gfile.c b/src/backend/utils/misc/fstream/gfile.c index 070ca1c649c..b3c9d755fc5 100644 --- a/src/backend/utils/misc/fstream/gfile.c +++ b/src/backend/utils/misc/fstream/gfile.c @@ -253,7 +253,605 @@ static int bz_file_open(gfile_t *fd) fd->u.bz->s.next_in = fd->u.bz->in; fd->read = bz_file_read; fd->close = bz_file_close; - + + return 0; +} +#endif + +#ifdef USE_LZO +/* + * LZO-compressed file support (standard lzop container format). + * + * Uses in-process liblzo2 decompression, following the same pattern as + * .gz (zlib) / .bz2 (bzlib) / .zst (zstd). The file format is auto-detected + * in lzo_file_open() by probing the first 9 bytes: + * + * - standard lzop container format: 9-byte magic + full header + block + * checksums + * - Hadoop Raw LZO format: no magic/header/checksum, plain LZO block + * stream (handled as a bonus path; only the standard lzop format is + * advertised via the .lzo extension) + * + * lzop header layout: + * magic 9 bytes fixed magic \x89LZO\x00\x0d\x0a\x1a\x0a + * version 2 bytes version (big endian) + * lib_version 2 bytes library version + * ver_needed 2 bytes minimum version required to decompress + * method 1 byte compression algorithm + * level 1 byte compression level + * flags 4 bytes flags (control checksum types etc.) + * mode 4 bytes file mode + * mtime_low 4 bytes mtime low 32 bits + * mtime_high 4 bytes mtime high 32 bits + * [extra_ver] 1 byte if F_H_EXTRA_FIELD(0x40) set + * [filter] 4 bytes if F_H_FILTER(0x800) set + * name_len 1 byte original file name length + * name N bytes original file name + * [path_len] 4 bytes if F_H_PATH(0x2000) set + * [path] N bytes if F_H_PATH(0x2000) set + * checksum 4 bytes header checksum + * + * Data block layout (common to both formats): + * uncomp_len 4 bytes decompressed size, big endian (0 = EOF marker) + * comp_len 4 bytes compressed size, big endian + * [d_adler32] 4 bytes adler32 of decompressed data (F_ADLER32_D=0x01) + * [d_crc32] 4 bytes crc32 of decompressed data (F_CRC32_D=0x100) + * [c_adler32] 4 bytes adler32 of compressed data (F_ADLER32_C=0x02) + * [c_crc32] 4 bytes crc32 of compressed data (F_CRC32_C=0x200) + * data comp_len bytes LZO compressed data (or raw data if + * incompressible) + * + * The only difference between the two formats is whether the magic/header + * and checksum fields are present. The decompression path is driven by + * flags: Raw LZO format has flags=0 so all checksum logic is naturally + * skipped. The 9 probe bytes (which belong to the first data block in Raw + * LZO) are cached in peek_buf and consumed first by lzo_read_peek / + * lzo_read_uint32_peek to keep byte alignment. + */ +#include + +/* LZO block buffer size: standard lzop default block size is 256KB */ +#define LZO_BUFFER_SIZE (1<<20) + +/* + * LZO decompression state structure (complete definition; gfile.h only + * declares the pointer). Same heap-allocated double buffer design as + * zlib_stuff / bzlib_stuff. + */ +struct lzo_stuff +{ + int out_size; + int out_pos; + int eof; + unsigned int flags; /* flags from lzop header; 0 for raw LZO */ + bool_t has_lzop_header; /* TRUE=standard lzop, FALSE=raw LZO */ + int peek_size; /* valid bytes in peek_buf */ + int peek_pos; /* current read offset in peek_buf */ + char peek_buf[9]; /* probe buffer (at most 9 bytes) */ + char in[LZO_BUFFER_SIZE]; + char out[LZO_BUFFER_SIZE]; +}; + +/* lzop file magic: \x89 L Z O \x00 \x0d \x0a \x1a \x0a */ +static const unsigned char lzop_magic[9] = { + 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a +}; + +/* lzop header flag bits (from lzop-1.03/src/conf.h) */ +#define LZOP_F_ADLER32_D 0x00000001 /* adler32 checksum of decompressed data */ +#define LZOP_F_ADLER32_C 0x00000002 /* adler32 checksum of compressed data */ +#define LZOP_F_CRC32_D 0x00000100 /* crc32 checksum of decompressed data */ +#define LZOP_F_CRC32_C 0x00000200 /* crc32 checksum of compressed data */ + +/* + * Helper: read exactly n bytes from the underlying file descriptor. + * Only local files are supported here, so this just wraps read_and_retry + * (same as the reads inside gz_file_read / bz_file_read). A short read + * means the file was truncated and is reported to the caller. + */ +static ssize_t +read_block_bytes(gfile_t *fd, void *buf, size_t n) +{ + size_t total = 0; + char *p = (char *) buf; + + while (total < n) + { + ssize_t r = read_and_retry(fd, p + total, n - total); + + if (r == 0) + break; /* EOF, return what we have */ + if (r < 0) + return -1; /* read error */ + total += r; + } + return (ssize_t) total; +} + +/* + * Helper: read one big-endian uint32 from the file. + * All multi-byte integers in the lzop format are big endian. + * Returns 0 on success, -1 on error (read failure or truncation). + */ +static int +read_block_uint32(gfile_t *fd, uint32_t *val) +{ + unsigned char b[4]; + + if (read_block_bytes(fd, b, 4) < 4) + return -1; + *val = ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) | + ((uint32_t) b[2] << 8) | ((uint32_t) b[3]); + return 0; +} + +/* + * peek helper: read from peek_buf first, then fall back to the file. + * + * During format probing, the first bytes of the file may already have been + * consumed. For the standard lzop path peek_buf is always empty, so these + * helpers degenerate to plain file reads. For the Raw LZO path, the probe + * bytes belong to the first data block and are consumed gradually here. + */ +static ssize_t +lzo_read_peek(gfile_t *fd, void *buf, size_t n) +{ + struct lzo_stuff *z = fd->u.lzo; + size_t total = 0; + char *p = (char *) buf; + + /* consume the probe bytes first */ + if (z->peek_pos < z->peek_size) + { + size_t avail = (size_t)(z->peek_size - z->peek_pos); + + if (avail > n) + avail = n; + memcpy(p, z->peek_buf + z->peek_pos, avail); + z->peek_pos += (int) avail; + p += avail; + total += avail; + n -= avail; + } + + /* peek_buf exhausted, read the rest from the file */ + if (n > 0) + { + ssize_t r = read_block_bytes(fd, p, n); + + if (r < 0) + return -1; + total += (size_t) r; + } + + return (ssize_t) total; +} + +/* peek helper: read one big-endian uint32 through the peek mechanism */ +static int +lzo_read_uint32_peek(gfile_t *fd, uint32_t *val) +{ + unsigned char b[4]; + + if (lzo_read_peek(fd, b, 4) < 4) + return -1; + *val = ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) | + ((uint32_t) b[2] << 8) | ((uint32_t) b[3]); + return 0; +} + +/* + * LZO block decompression main loop - lzo_file_read() + * + * Processing flow: + * 1. if there is still decompressed data in out[], return it to the caller + * 2. read the next block header (uncomp_len + comp_len) + * 3. [standard lzop only] read the block checksum fields (driven by flags) + * 4. read comp_len bytes of LZO data into in[] + * 5. decompress with lzo1x_decompress_safe() into out[], verify size + * 6. [standard lzop only] verify checksums of the decompressed data + * 7. return the out[] data to the caller + * + * Protection: + * - block size sanity checks (LZO_BUFFER_SIZE cap, compression bomb guard) + * - comp_len > uncomp_len rejected (compressed data larger than the + * original is impossible; equal means incompressible data stored raw) + * - lzo1x_decompress_safe (bounds-checked) instead of the unsafe variant + * - decompressed size cross-checked against the header's uncomp_len + * - [standard lzop only] adler32/crc32 checksum verification + */ +static ssize_t +lzo_file_read(gfile_t *fd, void *ptr, size_t len) +{ + struct lzo_stuff *z = fd->u.lzo; + + if (!z) + return -1; /* defensive: uninitialized call */ + + for (;;) + { + /* + * Step 1: if there is leftover decompressed data in out[], + * return it. Same pattern as gz_file_read / bz_file_read. + */ + if (z->out_pos < z->out_size || z->eof) + { + size_t avail = z->out_size - z->out_pos; + + if (avail > 0) + { + if (avail > len) + avail = len; + memcpy(ptr, z->out + z->out_pos, avail); + z->out_pos += (int) avail; + return (ssize_t) avail; + } + if (z->eof) + return 0; /* end of file, no more data */ + } + + /* output buffer exhausted, prepare to read the next block */ + z->out_size = 0; + z->out_pos = 0; + + /* + * Step 2: read the block header - uncompressed and compressed + * sizes (4 bytes each, big endian). Use the peek variants so + * that the Raw LZO probe bytes are consumed in order. + */ + uint32_t uncomp_len, comp_len; + + if (lzo_read_uint32_peek(fd, &uncomp_len) < 0) + return -1; /* read error or truncation */ + if (uncomp_len == 0) + { + z->eof = 1; + return 0; /* EOF marker block - normal end of file */ + } + if (lzo_read_uint32_peek(fd, &comp_len) < 0) + return -1; + + /* sanity checks on the block sizes */ + if (comp_len > (uint32_t) LZO_BUFFER_SIZE) + { + gfile_printf_then_putc_newline("lzo block too large: %u", comp_len); + return -1; + } + if (comp_len == 0 && uncomp_len > 0) + { + /* comp_len 0 with non-zero uncomp_len - corrupted header */ + gfile_printf_then_putc_newline("lzo corrupted block: comp_len=0, uncomp_len=%u", + uncomp_len); + return -1; + } + if (comp_len > uncomp_len) + { + /* compressed data larger than the original - not LZO data */ + gfile_printf_then_putc_newline("lzo block comp > uncomp"); + return -1; + } + if (uncomp_len > (uint32_t) LZO_BUFFER_SIZE) + { + gfile_printf_then_putc_newline("lzo uncompressed size too large: %u", + uncomp_len); + return -1; + } + + /* + * Step 3: read and record the block checksums [standard lzop only]. + * Raw LZO: flags=0, all conditions are false, so this is skipped. + */ + uint32_t expected_c_adler32 = 0, expected_c_crc32 = 0; + uint32_t expected_d_adler32 = 0, expected_d_crc32 = 0; + + if (z->flags & LZOP_F_ADLER32_D) + { + if (lzo_read_uint32_peek(fd, &expected_d_adler32) < 0) + return -1; + } + if (z->flags & LZOP_F_CRC32_D) + { + if (lzo_read_uint32_peek(fd, &expected_d_crc32) < 0) + return -1; + } + if (z->flags & LZOP_F_ADLER32_C) + { + if (lzo_read_uint32_peek(fd, &expected_c_adler32) < 0) + return -1; + } + if (z->flags & LZOP_F_CRC32_C) + { + if (lzo_read_uint32_peek(fd, &expected_c_crc32) < 0) + return -1; + } + + /* + * C-checksums (expected_c_adler32 / expected_c_crc32) are consumed + * from the stream to keep the pointer aligned but not verified: + * the D-checksum already catches any error the C-checksum would + * catch (a corrupt compressed block must produce corrupt output), + * and verifying only D-checksums is faster. + */ + (void) expected_c_adler32; + (void) expected_c_crc32; + + /* + * Step 4: read the compressed data block (comp_len bytes). + * The file pointer has skipped all checksum fields by now. + * A short read means the block was truncated. + */ + if (lzo_read_peek(fd, z->in, comp_len) < (ssize_t) comp_len) + return -1; + + /* + * Step 5: decompress. + * comp_len < uncomp_len -> data was LZO-compressed + * comp_len == uncomp_len -> incompressible data stored raw + */ + if (comp_len < uncomp_len) + { + lzo_uint d = uncomp_len; + int r = lzo1x_decompress_safe((const lzo_bytep) z->in, comp_len, + (lzo_bytep) z->out, &d, NULL); + + if (r != LZO_E_OK) + { + /* decompression failure - corrupt data or not LZO */ + gfile_printf_then_putc_newline("lzo safe decompress failed: %d", r); + return -1; + } + /* decompressed size must match the block header */ + if (d != uncomp_len) + { + gfile_printf_then_putc_newline( + "lzo decompression size mismatch: %lu vs %u", + (unsigned long) d, uncomp_len); + return -1; + } + } + else + { + /* incompressible data stored raw - copy as is */ + memcpy(z->out, z->in, comp_len); + uncomp_len = comp_len; + } + + /* + * Step 6: verify the checksums [standard lzop only]. + * Even a "successful" decompression (no error, matching size) + * can silently corrupt data; the checksums catch that. + */ + if (z->flags & LZOP_F_ADLER32_D) + { + uint32_t computed = lzo_adler32(1, (const lzo_bytep) z->out, + uncomp_len); + + if (computed != expected_d_adler32) + { + gfile_printf_then_putc_newline( + "lzo adler32 checksum mismatch " + "(expected 0x%08x, computed 0x%08x)", + expected_d_adler32, computed); + return -1; + } + } + if (z->flags & LZOP_F_CRC32_D) + { + uint32_t computed = lzo_crc32(0, (const lzo_bytep) z->out, + uncomp_len); + + if (computed != expected_d_crc32) + { + gfile_printf_then_putc_newline("lzo crc32 checksum mismatch"); + return -1; + } + } + + /* decompression succeeded, hand out[] to the caller */ + z->out_size = uncomp_len; + } +} + +/* LZO file close: free the lzo_stuff heap buffers */ +static int +lzo_file_close(gfile_t *fd) +{ + if (fd->u.lzo) + { + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + } + return 0; +} + +/* + * lzop header parser - lzo_skip_header() + * + * Parses the standard lzop header, skipping all fields to reach the first + * data block, and saves flags (checksum types etc.) for later use. + * + * magic_already_verified: if the caller has already read and verified the + * magic via probing, pass true to skip the magic; otherwise this function + * reads and verifies it itself. + * + * Protection: + * - every read checks its return value (truncated files) + * - name_len is 1 byte (max 255), path_len capped at 4096 (malicious + * header guard), path is read in 64-byte chunks into the 1024-byte + * stack buffer so it cannot overflow + */ +static int +lzo_skip_header(gfile_t *fd, bool magic_already_verified) +{ + unsigned char hdr_buf[1024]; + unsigned char *buf = hdr_buf; + uint32_t flags; + int name_len; + + if (!magic_already_verified) + { + /* verify the magic: read 9 bytes and compare, reject non-lzop */ + if (read_block_bytes(fd, buf, 9) < 9) + return -1; + if (memcmp(buf, lzop_magic, 9) != 0) + { + gfile_printf_then_putc_newline("not a valid lzop file"); + return -1; + } + } + /* magic_already_verified=true: probe already consumed the magic */ + + /* + * skip version info: version(2) + lib_version(2) + ver_needed(2) + * + method(1) + level(1) = 8 bytes + * These fields do not affect decompression. + */ + if (read_block_bytes(fd, buf, 8) < 8) + return -1; + + /* read flags (4 bytes big endian) and save for block checksum logic */ + if (read_block_uint32(fd, &flags) < 0) + return -1; + fd->u.lzo->flags = flags; + + /* skip mode(4) + mtime_low(4) + mtime_high(4) = 12 bytes */ + if (read_block_bytes(fd, buf, 12) < 12) + return -1; + + /* if F_H_EXTRA_FIELD(0x40) set: skip 1-byte extra version */ + if (flags & 0x00000040) + { + if (read_block_bytes(fd, buf, 1) < 1) + return -1; + } + + /* if F_H_FILTER(0x800) set: skip 4-byte filter ID */ + if (flags & 0x00000800) + { + if (read_block_bytes(fd, buf, 4) < 4) + return -1; + } + + /* read the original file name length (1 byte) and skip the name */ + if (read_block_bytes(fd, buf, 1) < 1) + return -1; + name_len = buf[0]; + if (name_len > 0) + { + if (read_block_bytes(fd, buf, name_len) < name_len) + return -1; + } + + /* if F_H_PATH(0x2000) set: skip the file path (chunked reads) */ + if (flags & 0x00002000) + { + uint32_t path_len; + + if (read_block_uint32(fd, &path_len) < 0) + return -1; + if (path_len > 4096) + { + gfile_printf_then_putc_newline("lzop path too long: %u", path_len); + return -1; + } + while (path_len > 0) + { + uint32_t chunk = (path_len > 64) ? 64 : path_len; + + if (read_block_bytes(fd, buf, chunk) < (ssize_t) chunk) + return -1; + path_len -= chunk; + } + } + + /* skip the header checksum (4 bytes) - skipped, not verified */ + if (read_block_bytes(fd, buf, 4) < 4) + return -1; + + /* header parsed, the file pointer now points at the first data block */ + return 0; +} + +/* + * LZO file open - lzo_file_open() + * + * Initializes the LZO library, allocates lzo_stuff (heap, 2xLZO_BUFFER_SIZE + * plus the probe buffer), auto-detects the file format and wires up + * fd->read / fd->close. + * + * Format auto-detection (probe the first 9 bytes): + * a) magic matches -> standard lzop, lzo_skip_header parses the rest + * b) >= 8 bytes -> Raw LZO, probe bytes cached in peek_buf and + * consumed later (first 8 = uncomp_len + comp_len, + * 9th = first byte of compressed data) + * c) < 8 bytes -> file too short to be valid LZO data + * + * Same calling pattern and error return convention as gz_file_open / + * bz_file_open. + */ +static int +lzo_file_open(gfile_t *fd) +{ + /* initialize the LZO library (version check, idempotent) */ + if (lzo_init() != LZO_E_OK) + { + gfile_printf_then_putc_newline("lzo_init() failed"); + return 1; + } + + /* allocate the decompression state structure (heap) */ + if (!(fd->u.lzo = gfile_malloc(sizeof *fd->u.lzo))) + { + gfile_printf_then_putc_newline("Out of memory"); + return 1; + } + memset(fd->u.lzo, 0, sizeof *fd->u.lzo); + + /* format auto-detection: probe the first 9 bytes, compare magic */ + { + unsigned char probe[9]; + ssize_t n = read_block_bytes(fd, probe, 9); + + if (n >= 9 && memcmp(probe, lzop_magic, 9) == 0) + { + /* path A: standard lzop container format (magic verified) */ + fd->u.lzo->has_lzop_header = TRUE; + + /* parse the remaining header (skip the magic) */ + if (lzo_skip_header(fd, TRUE) != 0) + { + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + return 1; + } + } + else if (n >= 8) + { + /* + * path B: Raw LZO format - no magic, header or checksums. + * The n probe bytes belong to the first data block; cache + * them in peek_buf for lzo_read_*_peek to consume later. + * flags stays 0 (memset), so all checksum logic is skipped. + */ + fd->u.lzo->has_lzop_header = FALSE; + fd->u.lzo->flags = 0; + fd->u.lzo->peek_size = (int) n; + fd->u.lzo->peek_pos = 0; + memcpy(fd->u.lzo->peek_buf, probe, (size_t) n); + } + else + { + /* file too short (less than 8 bytes) to be valid LZO data */ + gfile_printf_then_putc_newline("lzo file too short (%ld bytes)", (long) n); + gfile_free(fd->u.lzo); + fd->u.lzo = NULL; + return 1; + } + } + + /* register callbacks: gfile_read/gfile_close will use our functions */ + fd->read = lzo_file_read; + fd->close = lzo_file_close; return 0; } #endif @@ -1296,6 +1894,22 @@ int gfile_open(gfile_t* fd, const char* fpath, int flags, int* response_code, co } return zstd_file_open(fd); +#endif + } + else if (s && strcasecmp(s, ".lzo") == 0) + { +#ifndef USE_LZO + gfile_printf_then_putc_newline(".lzo not supported"); +#else + if (flags != GFILE_OPEN_FOR_READ) + { + gfile_printf_then_putc_newline(".lzo not yet supported for writable tables"); + *response_code = 415; + *response_string = "Unsupported File Type"; + return 1; + } + fd->compression = LZO_COMPRESSION; + return lzo_file_open(fd); #endif } else if (s && strcasecmp(s,".z") == 0) @@ -1330,7 +1944,8 @@ gfile_close(gfile_t*fd) * for the compressed data implementation we need to call the "close" callback. Other implementations * didn't use to call this callback here and it will remain so. */ - if (fd->compression == GZ_COMPRESSION || fd->compression == ZSTD_COMPRESSION) + if (fd->compression == GZ_COMPRESSION || fd->compression == ZSTD_COMPRESSION || + fd->compression == LZO_COMPRESSION) { fd->close(fd); } diff --git a/src/include/fstream/gfile.h b/src/include/fstream/gfile.h index 565a7d910c9..07ddfd9da0c 100644 --- a/src/include/fstream/gfile.h +++ b/src/include/fstream/gfile.h @@ -11,6 +11,9 @@ #ifdef USE_ZSTD #include #endif +#ifdef USE_LZO +#include +#endif #ifdef WIN32 #include @@ -45,9 +48,17 @@ typedef enum Compression_type NO_COMPRESSION = 0, GZ_COMPRESSION, BZ_COMPRESSION, - ZSTD_COMPRESSION + ZSTD_COMPRESSION, + LZO_COMPRESSION } compression_type; +/* + * LZO decompression state (incomplete declaration, complete definition + * in gfile.c), following the same pattern as zlib_stuff/bzlib_stuff: + * heap-allocated double buffer + EOF flag. + */ +struct lzo_stuff; + /* The struct gfile_t is private. Please do not use any of its fields. */ typedef struct gfile_t { @@ -77,6 +88,9 @@ typedef struct gfile_t #endif #ifdef USE_ZSTD struct zstdlib_stuff*zstd; +#endif +#ifdef USE_LZO + struct lzo_stuff*lzo; #endif }u; bool_t is_write; diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 72a27f363b4..a105f484373 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -72,7 +72,6 @@ reference if 'false' (always true in GPDB) */ #undef FLOAT8PASSBYVAL - /* Define to 1 to build ORCA in debug (GPOS_DEBUG=1) */ #undef GPOS_DEBUG @@ -211,10 +210,6 @@ /* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ #undef HAVE_FSEEKO -/* Define to 1 if you have __sync_lock_test_and_set(int *) and friends. */ -#undef HAVE_GCC_INT_ATOMICS - - /* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */ #undef HAVE_GCC__ATOMIC_INT32_CAS @@ -364,6 +359,9 @@ /* Define to 1 if you have the `ssl' library (-lssl). */ #undef HAVE_LIBSSL +/* Define to 1 if you have the `uring' library (-luring). */ +#undef HAVE_LIBURING + /* Define to 1 if you have the `uv' library (-luv). */ #undef HAVE_LIBUV @@ -430,18 +428,9 @@ /* Define to 1 if you have the `ppoll' function. */ #undef HAVE_PPOLL -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - /* Define if protobuf is available */ #undef HAVE_PROTOBUF -/* Define to 1 if you have the `pstat' function. */ -#undef HAVE_PSTAT - -/* Define to 1 if the PS_STRINGS thing exists. */ -#undef HAVE_PS_STRINGS - /* Define if you have POSIX threads libraries and header files. */ #undef HAVE_PTHREAD @@ -631,17 +620,11 @@ /* Define to 1 if you have the `wcstombs_l' function. */ #undef HAVE_WCSTOMBS_L -/* Define to 1 if you have the header file. */ -#undef HAVE_WCTYPE_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_WINLDAP_H - /* Define to 1 if you have the header file. */ #undef HAVE_WINSOCK2_H -/* Define to 1 if you have the `writev' function. */ -#undef HAVE_WRITEV +/* Define to 1 if you have the `X509_get_signature_info' function. */ +#undef HAVE_X509_GET_SIGNATURE_INFO /* Define to 1 if you have the `X509_get_signature_nid' function. */ #undef HAVE_X509_GET_SIGNATURE_NID @@ -655,9 +638,6 @@ /* Define if zstd is available */ #undef HAVE_ZSTD -/* Define to 1 if the system has the type `_Bool'. */ -#undef HAVE__BOOL - /* Define to 1 if your compiler understands __builtin_bswap16. */ #undef HAVE__BUILTIN_BSWAP16 @@ -703,15 +683,6 @@ /* Define to 1 if your compiler understands _Static_assert. */ #undef HAVE__STATIC_ASSERT -/* Define to 1 if you have the `etcdlib' library (-letcdlib). */ -#undef HAVE_LIBETCDLIB - -/* Define to 1 if you have the `__strtoll' function. */ -#undef HAVE___STRTOLL - -/* Define to 1 if you have the `__strtoull' function. */ -#undef HAVE___STRTOULL - /* Define to the appropriate printf length modifier for 64-bit ints. */ #undef INT64_MODIFIER @@ -886,12 +857,15 @@ /* Define to 1 to build with LZ4 support. (--with-lz4) */ #undef USE_LZ4 -/* Define to 1 to build with MDB locales. (--with-mdblocales) */ -#undef USE_MDBLOCALES +/* Define to 1 to build with LZO support. (--with-lzo) */ +#undef USE_LZO /* Define to 1 to build with Mapreduce capabilities (--enable-mapreduce) */ #undef USE_MAPREDUCE +/* Define to 1 to build with MDB locales. (--with-mdblocales) */ +#undef USE_MDBLOCALES + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES @@ -938,7 +912,7 @@ /* Define to select Win32-style shared memory. */ #undef USE_WIN32_SHARED_MEMORY -/* Define to 1 to build with ZSTD support. (--with-zstd) */ +/* Define to build with zstd support. (--with-zstd) */ #undef USE_ZSTD /* Define to 1 if `wcstombs_l' requires . */ From 259747de7a748d250d056d488abe06766fbc7f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=B9=B310304955?= Date: Mon, 10 Aug 2026 10:32:48 +0800 Subject: [PATCH 2/5] Enhancement: improve lzo_file_read logging to display detailed error information --- src/backend/utils/misc/fstream/gfile.c | 34 ++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/backend/utils/misc/fstream/gfile.c b/src/backend/utils/misc/fstream/gfile.c index b3c9d755fc5..d2304c9d64a 100644 --- a/src/backend/utils/misc/fstream/gfile.c +++ b/src/backend/utils/misc/fstream/gfile.c @@ -463,6 +463,11 @@ lzo_read_uint32_peek(gfile_t *fd, uint32_t *val) static ssize_t lzo_file_read(gfile_t *fd, void *ptr, size_t len) { + if (fd == NULL || ptr == NULL || fd->u.lzo == NULL) + { + return -1; + } + struct lzo_stuff *z = fd->u.lzo; if (!z) @@ -502,25 +507,32 @@ lzo_file_read(gfile_t *fd, void *ptr, size_t len) uint32_t uncomp_len, comp_len; if (lzo_read_uint32_peek(fd, &uncomp_len) < 0) - return -1; /* read error or truncation */ + { + gfile_printf_then_putc_newline("lzo: failed to read uncomp_len - file may be truncated"); + return -1; /* read error or truncation */ + } + if (uncomp_len == 0) { z->eof = 1; return 0; /* EOF marker block - normal end of file */ } if (lzo_read_uint32_peek(fd, &comp_len) < 0) + { + gfile_printf_then_putc_newline("lzo: failed to read comp_len - file may be truncated"); return -1; + } /* sanity checks on the block sizes */ if (comp_len > (uint32_t) LZO_BUFFER_SIZE) { - gfile_printf_then_putc_newline("lzo block too large: %u", comp_len); + gfile_printf_then_putc_newline("lzo: compressed block too large (%u bytes, max %d)", comp_len, LZO_BUFFER_SIZE); return -1; } if (comp_len == 0 && uncomp_len > 0) { /* comp_len 0 with non-zero uncomp_len - corrupted header */ - gfile_printf_then_putc_newline("lzo corrupted block: comp_len=0, uncomp_len=%u", + gfile_printf_then_putc_newline("lzo: corrupted block - zero compressed size but %u uncompressed", uncomp_len); return -1; } @@ -581,7 +593,10 @@ lzo_file_read(gfile_t *fd, void *ptr, size_t len) * A short read means the block was truncated. */ if (lzo_read_peek(fd, z->in, comp_len) < (ssize_t) comp_len) + { + gfile_printf_then_putc_newline("lzo: truncated block - expected %u bytes, got less", comp_len); return -1; + } /* * Step 5: decompress. @@ -596,8 +611,17 @@ lzo_file_read(gfile_t *fd, void *ptr, size_t len) if (r != LZO_E_OK) { + /* + * Decompression failure detailed error report + * Common error codes: + * LZO_E_INPUT_OVERRUN (-7): Input data truncated or corrupted + * LZO_E_OUTPUT_OVERRUN (-6): Output buffer too small + * LZO_E_NOT_COMPRESSED (-5): Data is not in LZO format + */ /* decompression failure - corrupt data or not LZO */ - gfile_printf_then_putc_newline("lzo safe decompress failed: %d", r); + gfile_printf_then_putc_newline( + "lzo: decompression failed (error=%d) - block: uncomp=%u comp=%u, possible causes: truncated file, corrupt data, or not LZO", + r, uncomp_len, comp_len); return -1; } /* decompressed size must match the block header */ @@ -656,7 +680,7 @@ lzo_file_read(gfile_t *fd, void *ptr, size_t len) static int lzo_file_close(gfile_t *fd) { - if (fd->u.lzo) + if (fd != NULL && fd->u.lzo != NULL) { gfile_free(fd->u.lzo); fd->u.lzo = NULL; From 15fa84c4e50fc335b7dc3c841f1b966fd04e3eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=B9=B310304955?= Date: Fri, 14 Aug 2026 11:39:14 +0800 Subject: [PATCH 3/5] Eliminate double references and adjust LZO buffer size --- .icodemate/icodemate.json | 7 +++++++ src/backend/utils/misc/fstream/gfile.c | 3 +-- src/include/fstream/gfile.h | 6 ------ 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 .icodemate/icodemate.json diff --git a/.icodemate/icodemate.json b/.icodemate/icodemate.json new file mode 100644 index 00000000000..76e1fce1964 --- /dev/null +++ b/.icodemate/icodemate.json @@ -0,0 +1,7 @@ +{ + "plugin": [ + "/home/10304955@zte.intra/.icodemate/market-plugins/plugins/superpowers/.opencode/plugins/superpowers.js", + "/home/10304955@zte.intra/.icodemate/market-plugins/plugins/usage-recorder/.opencode/dist/index.js" + ], + "$schema": "https://icodemate.ai/config.json" +} diff --git a/src/backend/utils/misc/fstream/gfile.c b/src/backend/utils/misc/fstream/gfile.c index d2304c9d64a..f037c899122 100644 --- a/src/backend/utils/misc/fstream/gfile.c +++ b/src/backend/utils/misc/fstream/gfile.c @@ -308,10 +308,9 @@ static int bz_file_open(gfile_t *fd) * LZO) are cached in peek_buf and consumed first by lzo_read_peek / * lzo_read_uint32_peek to keep byte alignment. */ -#include /* LZO block buffer size: standard lzop default block size is 256KB */ -#define LZO_BUFFER_SIZE (1<<20) +#define LZO_BUFFER_SIZE (256 * 1024) /* * LZO decompression state structure (complete definition; gfile.h only diff --git a/src/include/fstream/gfile.h b/src/include/fstream/gfile.h index 07ddfd9da0c..ead06377892 100644 --- a/src/include/fstream/gfile.h +++ b/src/include/fstream/gfile.h @@ -52,12 +52,6 @@ typedef enum Compression_type LZO_COMPRESSION } compression_type; -/* - * LZO decompression state (incomplete declaration, complete definition - * in gfile.c), following the same pattern as zlib_stuff/bzlib_stuff: - * heap-allocated double buffer + EOF flag. - */ -struct lzo_stuff; /* The struct gfile_t is private. Please do not use any of its fields. */ typedef struct gfile_t From 7fa36abb9b39519ab2d31072fa35b7cafc222ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=B9=B310304955?= Date: Mon, 17 Aug 2026 09:53:31 +0800 Subject: [PATCH 4/5] removed the invalid files for gpfdist supports reading LZO-compressed files(PR:#1887). --- .icodemate/icodemate.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .icodemate/icodemate.json diff --git a/.icodemate/icodemate.json b/.icodemate/icodemate.json deleted file mode 100644 index 76e1fce1964..00000000000 --- a/.icodemate/icodemate.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "plugin": [ - "/home/10304955@zte.intra/.icodemate/market-plugins/plugins/superpowers/.opencode/plugins/superpowers.js", - "/home/10304955@zte.intra/.icodemate/market-plugins/plugins/usage-recorder/.opencode/dist/index.js" - ], - "$schema": "https://icodemate.ai/config.json" -} From 1991da53ad172be97cd1d080abe946f57fc56be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=B9=B310304955?= Date: Thu, 20 Aug 2026 21:08:47 +0800 Subject: [PATCH 5/5] regress for gpfdist supports reading LZO-compressed files(#1887). --- src/bin/gpfdist/regress/Makefile | 19 +- .../data/gpfdist2/lineitem.tbl.long.lzo | Bin 0 -> 17702 bytes .../regress/data/gpfdist2/lineitem.tbl.lzo | Bin 0 -> 13937 bytes .../gpfdist/regress/input/gpfdist2_lzo.source | 184 +++++++++++++++ .../regress/output/gpfdist2_lzo.source | 218 ++++++++++++++++++ 5 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.long.lzo create mode 100755 src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.lzo create mode 100644 src/bin/gpfdist/regress/input/gpfdist2_lzo.source create mode 100644 src/bin/gpfdist/regress/output/gpfdist2_lzo.source diff --git a/src/bin/gpfdist/regress/Makefile b/src/bin/gpfdist/regress/Makefile index bf7b0a9ad1b..91969c9f637 100644 --- a/src/bin/gpfdist/regress/Makefile +++ b/src/bin/gpfdist/regress/Makefile @@ -15,6 +15,10 @@ ifeq ($(with_zstd),yes) REGRESS += gpfdist2_compress endif +ifeq ($(with_lzo),yes) + REGRESS += gpfdist2_lzo +endif + REGRESS_OPTS = --init-file=init_file installcheck: watchdog ipv4v6_ports @@ -33,7 +37,20 @@ ifeq ($(with_zstd),yes) for name in `seq 1 1000`; \ do \ head -100 data/gpfdist2/lineitem.tbl >> data/gpfdist2/lineitem.tbl.long; \ - done + done +endif +ifeq ($(with_lzo),yes) + # Prepare data files in case the lzop tool is not available on the environment. + # The LZO-compressed files (lineitem.tbl.lzo and lineitem.tbl.long.lzo) are pre-generated + # and included in the repository. Uncomment the following lines if you need to regenerate them: + # rm -rf data/gpfdist2/lineitem.tbl.lzo data/gpfdist2/lineitem.tbl.long.lzo + # lzop -o data/gpfdist2/lineitem.tbl.lzo data/gpfdist2/lineitem.tbl + # for name in `seq 1 1000`; \ + # do \ + # head -100 data/gpfdist2/lineitem.tbl >> data/gpfdist2/lineitem.tbl.long.tmp; \ + # done + # lzop -o data/gpfdist2/lineitem.tbl.long.lzo data/gpfdist2/lineitem.tbl.long.tmp + # rm -rf data/gpfdist2/lineitem.tbl.long.tmp endif $(top_builddir)/src/test/regress/pg_regress --dbname=gpfdist_regression $(REGRESS) $(REGRESS_OPTS) diff --git a/src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.long.lzo b/src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.long.lzo new file mode 100644 index 0000000000000000000000000000000000000000..fa2d5e5bdb2047644c39a6c7cd0b5ec6a98d128f GIT binary patch literal 17702 zcmd6udwdh;mG?E0ZH$mC6G#fihVe5q8r_+@MqJ9xfSs62zz|4CDq+c%6Jg7?Ok_HM zNJJ_Uk%&a3A{ALDb+V>)TBmikrgd6p>u%_lY@L?wl3iy@TGEhRw~CCZM@Im-C$SJ-^?v+IzQXjQTQtDT8}wuf_^>8V#n=9O-`Wu^mks_%T1! z)EsDv1opWidqb|!ffiR}U-)O=m@}-wzAtGsRilr6^vi{sVpd^ANf8wI-3?zJQE-W( z!nhbAtTA%RW@U@QdOVVo;hijh4#PMn%bmkCZr;gpO4a(>nw{&n;WeA_t?RdK-?CY$ z+qPrfT}q&_Bjjtt>svb7BJH?uZ_EAw9tj5YtRl&ZN8uD+Qg}fY85b*c8a=HV$@&&1 zSi029bMpBT%{sx!F-qN-?qMP$F|MNrZqLSN~jZWXlnL_aDN~iX!Zx1 z>ze}Y%kcJ4C;ZLcmXIG;?;|Lp!Yi`kcC(yIkl~G3rpd%vuV5JK_66+O~cpzNTiI($>_{0oQWjZQV;W^Bs5~8g2`;x2t!9>+*^^Se&eIA}0$jhEa#g z<2N~ersLR)>P30x95zdIvXZiS%jWfp->4}M!8EkH@P<%FLqiihWMF@w8LkJH<3xBw zQQ_c{9LF%Z2YqLeV2$dmehejy+ctSsykRwFO!`0_~A@S4c3n zv^T*F?`;eC>VvpX&wH)Ka5X_x1jfyYKa!+qxt3A3DI0uc!cPpak`B;LCS`qA%kh0y zzf{uF+}Pga58y2g>H@(VI!4xOO5l1vSTVl(`W8K}z(Qadcu2tm*JWhJ<*`Kc^f0D_ z;bEOjdb>JK+`~S&I9yf`dvt|(M{`GehcEPqv7=e3&=kRd!-ZJR1EUQWgjaUEWy!_G z4?k7zR@TtRb*!?5&S06trz$wkGE2L?x@M~q4aYam!5{WL6ma42ah_2`zqdePWeMi| zgn{j_N|%kXGJ19)JF+2eyt4h3CpxX;bN*82R~BNjx&WmDxF5rd?%`9ZGFxFGcj19! zbJ%p~7{;)QDeNosH3r)8W?$pb_H!i~E!;#>*Q}&)@Fg<5$0c@@u_v~3)*U)#^w^Pd zHgW2x#7M(Mim|vo6bOWIUqd9&c5*{~v|g(h6xq{R#VXbU&Lg>D*`GG@O;-8+y^??L zON+#4RrlW4v@(0mIG!nT;;kJ`^$&$QCo)<*?5lq$yCgezmJ4t4hdTA5!tUG31m7%z zZ$y@tTw;cX)DxA3M(** z6v`I4VKqa1ak24|l!m^D>0tdrNXhi5-@xXKS0pelIq!u)*jimUUXf|j=^^^b{40Ou z4#hn3=+)D9-dZtR91S}b;1P@&n45IYZpXt-&5;&-Z@}05!{xq4U%LYqt zta$`hbaDF{m{f)B6FnDw+x*8$=J=D+{#RHJa}i5C8MzQP%!XHtG_^Fh!wo&kd$W0X zNm$ozx8P#^N6xmptw(k2h&5Dp&XF+AVR&WTmfG6&>+0}zTdK%ZCGEJq_s!0Ql2X_6 zUs{jbBe_ogj>)zzILCE@j*Z%W6PF_zPM*KpmK*k>q&aSr+#8$<0zkyE0^=zBVk zM`m5_bn{7?wcZ88!J3y;Lex$F>{uDgT{XS3zWJXG0X%*@_K~ist);oCo_@I8h3{=G zm1Dn{1z{V)s=zbc`H!=*?fqhQxc#V!%lMC`IJS1Nf8*>2TSBm10?~Sy-l2^ao7NYy z&aa492L8pZysqUujLR+j%EmeSy3Q<@-ga?Fw$tHKtO<5MJv0x~Lns@nN8dENMQ;38 zF#~;6F9wsZl!@L)y4zpVN=#gT#?hYm?!~9jOP1EC5o0~F68mcre2f=afphiyPMv11 zE=wB*2GRzyih_mWwrBOeCqwqTb#2zdSq{Wwi^l1g;de%s zIe#Rr<;RzVP4s0u@)BsUvj2Pgu{J-dwd!E@;X%En$_Ir4oCCbXa#Wnkn&<_y<3_R; zA6R_qUUA>+4E)crf0_6mIgf#@Rd0|bB~&#_fRFIJ>~@K1Thc?1I+F%6&h}tL75=1w z6~wB~6|{Hm3rAWa)E?Z`q{FS343ScdB$QDa2tL~sF_5<7aGUGbQSZkp{5 z{i|)^ZQIbetq5;yYJdr^_k|iPLnTEnJr4neQ{0gs7r{~EkvSg1d;2My@X*|H)%c&UtB@?U2-aCt6P#L$#yF4i;917O=F^qLv}=@I-=iLB(MM%gdn+p~G5O?k*|gft@!{9@C>FB3d0>*g|UT>iG_F zv?RK-$c48@0=`frh_;Hv4`f&|okqU#8!$#_J_id0wo>wYF`KQ$7R{!5u0_ni2;9NBv&E)yAs8j&yg zgJ=Caf^F7wJO-yA9{GAxi3e5FuyG-e;aIoKlEvJ>33|W~FX^!#!`RH%&S0Fyv8exR zq}3e`&C#-MMsX8YWSt~%tc%I6O=%oKjJznP7NR=^kLT5eqUAxjIJtWn{?)JPc@ez` z0g3ovIV*AI=*rX_y5RPKk?faFjW8+mV?ShNrn)%rGY4Jfh&08Y#URv5X!$Z2^3<~! z%e#bFXEH=b7Mz~LnSYMppi7XiY29qq+$Rq}GQwJi7tuc_;lzBhoMST&6uK2RE4W>< z@XF z!=Z_z<}Uo|Ix}N4r)+~`-pAWaESIzuD?H>G3?sT-QdUlXi*~F?n@DD9PYJO=JgMc_ z;*x}&7_K7rY_Vvy^}b^zeD>HZ0n!5xCwg2$LOxm|PrQ|!LmDgzth@DuiIJ~vBOP0m zHwyKTWwCBai6;wSnY$riQSqgj7TX)e9_rL8DN>KBcM!j=^PuuAwyKIF;dbP3Ha{Nf zq`URj^rAxCw@(ib$0M%=_A^8i-s9%STI0(gaP=Gibvw(^4!#NF>@h>eN>^66t{tv7 zva&nzXGE^UAiI+;xD&E9JQt1Cyl-Ss@v6*DtFS8kGBvnb=c{jPX>X4m?;GlB?=VW} zii_MM=>(C7fSrCKrLoPq9S-qhNr@;}M;t{``d#VN%AoDHT%f_ovBbq4MDI5AAtjxV z+{mzoX67aJecN0wW2|7;6vo{a3%dBz@0USdeXb5^Y?wz8#gIH3@|%e3 zDI~7BdJkculaVNL*K1mjJyIC->mUIO_}b||-{HX94jVW{M6ARj*m(2hxFL0r?x$n= z-b_zfy7i=%=Pp$w4w=USWVhE6(FMPQ9G0U?>SI6A>Lmz0Xf=(>EwfxQzQ5D$Z8=fuMsGVgG~z52 z1V(_o+a2B?UutW*mm7H}S=#pwY$Jz*4c<#TXf0GTF!4!y>bqLlcCu_6TTZ_|&mFAk zU3SXPC(2>xP$L_zeoQQo8g_;{sqy8H8(4+#u0q2hjAcZQQtnSIqc0aH=8zMsiQ&Cu zsoaB6vh#}Hp_xUEyZ0JdH&MD0troPrn{mmBWcoe$=Uw{j-UJr9Eps?-k>!A>qQRe=@ zqi)*IXUgc`S3+(Zv^ITDPCb<|s>4;=60=c}eU6Lw5q))U)ws;jRnZdkZv@ne#6+FW z1NDJRjxT~(9MzpRa+&c2M)p)6GcnW(>&f5X-6FC@+XCic!V(q9;a!mu!r>-ExQ=I7 z*2TwavWIMMV=|h(g+(DbxFmKLs(SzxmGW zG!h5v-N~~$P7*|yJLAc8(SvkvS+=zY%lf$jz5`_kU>qLWf@RSV|Yo z0$d=-Za44ZVm0x1Y}$pgvwINLJ}^{Hn41{6tDNk^&HYqCAlwx3g@Z&8{&avikH`KG z$~=%Vq4P0hc^FmQ14uxEi;R3gyw|>l_I1cq=z%^{eDN`iAKy>49cjnUT&_=zM|Pq^ zcRKKtkz#723v_NtkRkUeKWS&o3k4w_CswcR!N!cGb3ena3 zh~etEKpMDM~;cwl{mj!>GKm;sG`k(m9Uj0z4U4=(M*M0D>wR^>9%Q{_@-$ssi67Ry6n}0M^VvbjalUl|t!H&1Z`fXcvM3?uq5}rDIWW5HlFhHS(7bOX}85SoAnHA2lV5c5N*sz&EwAtUmnjcKQ?Z;5&ID_H|g(H!I* z?AF^+`Gc0)6T=s*7d8eTJ!$eBY?FxLJvuiB!Cgq*Ll|(|knY-%lO`_tizk;r%o-{j z_}E+!!hpqi(9n*~ayE7r<3&**_tcPshi$LSgGI!|e@1ry&DnFJ8(3l>>~xw3Ye|MV zQo^H1E9!qp2LYOq`BB@EBa(Gw9!mN;b_&F>g}B1^n&g6QmaVfa(aOYA7(hUM+jlvJ z9Dhs;>yn4ms_OJI?n4tFY$vy_fTAr*^3huD^W44?5uJU2i$9Nv z08XfkhgiYT(ZbZ6*WctL0g`F#*SddYXsWk#GJ8?_0c%g8@`F~B8+DyjeqRW+s3`L; zKJk-y4L!fYU#+}Pywt)(u!u|1pas=S7tAM;41sriUEVB68CTa^ z@s+kHG7lw{$7o9>D%~aTnBy|zlDRldI^?0%51F7=Mtx7Bt*saj)f+XuDfTkG^$W3* z*H;MW@_juI#}>2xN2qR(`9)8t(~9q*`tBy3PCc$-=>KIu{vrA;jo!sbIl`^}!cQS? zNW|PuV4N5Rc`lG{&1(3gI1|4p+f@8KU1Y1I18VI_q&i(}t1xxC+Roj_lfT&-WrKH4 z0kBF2`14Po5@vHOnvv}TnC6Z=IEN}cj~IZ2PJ(DWPhoP54sc=IR06fRiqNNvb1q!+ zv_@1cAE=}^V=Pd4G2kwGu9kDWk8Sc+XmdE;Ybpd(mSfS1p#*Am6<`3Q6m#Xe7&V;G z@N6bWow;@kESw%pQ^tQ9!mE&cx_DP{c8<0i_0xw9-Yh$R;lOoH0WmO?0M)q);Rdo)lesVGy$zR;FX!7XeI=t%UIU@&9u z_clU#pI)znIO%b+F;j`01HqHPUFG=R@`$LJt@2~2V1c|tF`5U}YU%F_0o5w;C`Q2F zQ?CT{@CPiAYarZlVh(fd4h&i>+>b+2d`rq) zkZ-rhOrj{X0E0pxFuXNK$R06DoYx2?Y7RZAkRZh!56RY^TvLNn7W5bsWCGIw^ZIme zruRTMn_e|L$NvFd=5Xs5@P=b1E>b>=hc}u82=8Hn{Sx+SrDIz6fx_tbkLsEC?`ACr^;~esHu|}5Sn7R=*R}b!Rn$1e zrE^F@#q!2klSsiOGhr`4h5P4omh3_;>^j*wO%*V;%M*Oz>N;J%Pm(H09nj?)R<~jd zr)9CJlpFunNfVRY)>D#QHpLiH|CBK12t%SI=ui!;&=ahj-?R?q<&6#ff zz`Mhv#Q@c=V}_zh%m69XqC98#(SJz$>sj}?oe&x{>#$lV!Bn=T`2^2!`W;qHzgEq{*-P5Qx9(xxOKgOk#?@H{YBHw7x zY9sN0-w0(HB$_=q07}jST-2pvhw!$WsYBA5DjN9dU|D2T2el0J&3b4OjQKvCQ$ zi;=p5N%U}}j%_MsE#H|7vH3R}SaxJ0Hjhiq3;qX?hc<&s9%iXHDbF6_hbPfPKR{Rd zRM>s=J(W5Ht^XZ#pePc!ChDDlqWL(mpG!qKuC&1(8-36hap%*I1-inBgVseJXV5wcY|cbjx|@u**IL z9SZZ%!G-_JcOcj(1}$I*g%8_dPCbIX&y<4>-^hs0;0){lI}(z`Xpb`o9-J23zohH4 z!NZZoviS|WfCP$E(L*tnd>^PG!wA%oxg8`!T>b-)hu4Fj1s?njN0+*4W&G`>e{!_rpUotw z-sSNn}oM_QMfgi$SeUYkxF$h`OQjbd|R04qjPSg62#p!I}Ig% zTPdThG>KNq&M;63m@&5Z`IwDJuKwsr9EI6}$!3SG(9tm@^~Ws zn_-a}@C?TQP5h}wR&FK}4Jw&<_$=)IZsU==WVFI=PV?Cc>UWYq5PE%fzU4p3C4Rfs zM`T1a>U!Sc5o5c_pFzuzy5_ivNc5Kikuc@DQti-boTL(XgSpmu*e>%*QtcUXDwlAv z7GY72OZ4}`jK05If^Ed$p6m(cMkqnxpD$9I!g(l>ap$1~z6PDck3wz>(! z+Nt1JY|GPLTvS@1os9mF|a7}P7h|qXRBCZk@>>f&_7@NVEjY#LwDYq zN?;3R>Rn2RZ0ss0{uQZIVz3LKMByZqXh%Vuz$N8%q;d&N?E(UdCPrwItexf(YC0uZ z|5PSB$!zs?GI4o>`K5LBSGW+?g zVG2xSwvz=6aEZc6E&(_XFMv}V$OJ=;%ZVS^B#L$Rd`xa7uJMF}Svs&SJd8oBG+Z&U(Gl+z86$6<9o0U$Mi&k$jabd%Y z>v(6>|6~>GbOV#9Tb) z(ZWVF<}hcN&wvwlp)ijo@NKrp#uFvYgs`5dcIhDL1wsK~9fUrz;@zVbnlm8F*ej;(O?z>$>6T!$6GCt#-#dmsX` zd0wI2>cX?bM+_3l?9f88=8?~D$N>vW?If_cJOfy0)sA|*_gjT9_&GrXG2eF+x`;1Q z(S>HSiZ0f8^5}w%&rO>0@SvWQBpC5~K20#}qds$g%V!dd@NMYTeN4(tjP47?^AID-5Z4(-xC>=-OY264L5rsjY(XVV+7!g-L>xV_4gf}~@A0{SUj}bG+vuWG zej-AfRgjVI`b$B`YgM8HB;!Y$L}&{F%Sao{Z)^ds_|PMZK`s!#1C8U|Sjmcem6N3xTFZ*I=q1 z4IQ>a!CwC?h~B7OQtD@P?|h_evAlqBXdDsQdr1Iip_7O%JRkM>i75wF!*A zP@@tIPTX}Hs`ZfJyU@_So=;}a)R&>Fj|NW^PRYxtYoE`)KyQ77u&|`Ro`_q>M77#E zaqRI+*3mZ9Z8tAiSpa}Sm6H)$Y@n|~yQ&bgy>LoPMuuVoZu&-NakMJs#`V!>RAQ~gS{CPNThdvql9%#n&h49&)ySnV>wx9jlq-yNoLb>CM zBD&NGeGbr!QLCewQ+d%CG{wf+rbJ_yDjFjUU+4nO7_`b2so|KUpg}*T3dZ`s(=E~C zcPu4S*}7xayi$xBpv_+AOMlW^MjoF93CpB7Eagl8k?z}=IoZ947>RZ7?bj~Tu5#i~ zq#?r*^Xd&KWmPp{1(TXEG``+EmINZ6Q-o1hxa1M*;x#N|saPY^7ESmfwYvRKy($N@ z{J0FC(t}~D9*lC}*(ALhA+Ip9o!DAKPHgViQu1Thg<$93xKeITto-R8Kzc@XZ!DP9 zewks^7A&<{WE9)1oKkgP#9M1QE^*)9@<0m}K54zAr_w7iRrp1GvY91DY0%xtB9)OG z@4&|Dv63%rT#z&hf*7m|1he4mZ;?kU2kDm z)zkQ5YS1$NYeZTL;K`tc!lE~J^QiH}4Wh2Ni7fxZN#LQiiz7?W@fH`V?IM8rZnRjm z_QC3SC34iFk(D}erlPB7IdN@6dak+qK(vHPCAyx!rJRcxIC$)^h`jd=Y{(0`dK|}k z@Vua_)5xL%&ihw%@860n@*c=~5@+b}>Aae&@7HN9TgsR(Kk;f=8#34k$BWi$bDK;8 zdbf^8AMl{$vL*J@f+J=aGotuTVzdsT0f*C()AZG_S=DiMU~R;;$H?xls}e4vu_LeF zB5EDi1zf~Yzle(Ck+g06B~Wn9Wx}WIa*5{ zH?Umw%24@C{Z>$C?z+?4c%Yo+(AJkRU{)eJo@C+yRl$Y-FT>*M<9r{`xB-Y|u9dHn z(LAhFyZJ64bg(;s8AZp&%5bE4-w`>Xfm5VmBZ`HW1r!X#h$M?%PXXv&GgXT8&(=2pSr`|aw6vpWbm_Dx>n6B1MYPGJy?CENq z^O{auCF^*Ok;@QlR3$kK0E`3AjstehH3-@R)6`Q{zWW@3W4PnIL)Q)*x3%LN&`JzB zLHNQ!fdpANi0d+?ATO8)*q$9zUz-&!mgWczw?I3z)C5DhwDCEVH$$B;u7BRhNtP>E zZ1>|PMm%HM6f_NsIyly$F!|_C2VGY90WMLAqN3scVX~cviGi7 zd#u`Wt_alYQ_BB52AJ%U_@b2{I0f>n!hlD1b4+BRmSKU{HaXp`8rFKIP`c1j0GAS> z#}h8hQ31`YPh){erGCa}0yd!bG}NO%PX{EwF|S*&ckN7IT{WOuV2AYC6NOXVhOr&m zoVYosUPvuBK#NI&pHrm3wr95O&+s;E`uTV{ZPm#BM$7YsEm5ys1H^dB#sF=aT-PBd zV<=Ho%_?@E!3<8kB0Ic=+17zYRy12ulE&1cffl$iU_c8w2U0p;yCtS80CoU;OHz)^ zhR;sR8jjSkA%_-}@5+XQk`Ua`J=lXKE7Zo32S&NCWUy=e@SG`-pu)tqz+lsfZ#4x= zb08sQ@`k-Xn5A~0@(PDU&CycH{t?E-7f(}!(WeR~`ySR24CF*1zXN~T$jiwe+^BH? zl%RSIxSuYDU&o9Z)opN#u2cA}IXQ^#z*W-+u-yI|IK#Q$JOiO=8@;1YQlJ6h@qqr} z)qfKL&#?B|(is{EWt*xw16d(7aX-kvq2eKmsGBv0}zog7Qn%-g7rxhVL7Q2?I(5IM--)DkDv@vrndC{IXstBxR4tud6lpkB$A?^5Is1$B-xtsDZb0^u zNvfg=7~q^}!p*3nK2?oEh>MSN!+dDEc=%5)O!y3 z=w9JgKw?e#B#tSVF7z{yQ%oWWi&A){XQq}SS{eD_FyIK#JpqnTN&JmC0;Cjs zTq*aNVY%Z3%Y%|)HyyGJ>$@)$#w7=!?@Erj@cF&aDBFf|&h*N?K zfE9htX~9M8N@}qxxcJ9|@3SMO!5S4-h=&(~njZBy=q_DOYN1tyC#u@Q`^l_a_c6yT ze!c|qBI{<55u#c=wE{(Pf|bO?Cn~X+6kL1^f{XnnEa)v7{*z^f-b&wVJAgQ6_e$ll zb6@JgCe;=&897MtAgC?SaS%3TSLD`&vKTJ7qVP9^1@>F;d>#tXr^=aV}moOC#~c}`KZ3|9I0Oip3G#I!!-d}*_kI+^_`ydx(ArVv(Ps!b2o zE;D}?sd#0+lTD^s*&~DPI3F zXU%hpSN2E;?#lV>fK#BlRbeMZ@|NQDa88Buq@w;lFw~jTYuLAST>4nt)bmC3cWa!J zpklQ-2P%rDJ#%Pixi0USyOC5JwdY9%C{fXFt^tjCaKqa2Z zx_jU4foPzTiXrz*QsI9QbmUXcIG`1&cKYmEcbM;~N??&D>OVH+JaOLtL|t2eRz#km zMrw*@nBh?GmK$Nk^Di)pr4le|nvA!D1yjvJ9hYKsm&40$0y3ly)?O_|-2<<^1QQBpt$ z9i9P0<}g{yg4qq21*Y05k8Z@#B(~T$Arb?3>GDpulws%-s&94P=w>mlWc}YHt}w6s z`8u_Td{ND3C#`Paj93VA@q$A0o^2-M9BxYRj_VG$Nr;hci546jX0KKBYk6pTR#Z9Z)w?cc zEWOSfOl{3u86|j5cT0q!I@-|C()hCf7GkVHLfB5f?eg{RLg##Rep!Gt8(Y1Fu6VVbFlilc-_?fDM@$uzJhs@|8EkjPB*r zFyo?XjAaB=u(AktKYR>tZ9jYOMjV`Iy~#uj^%o;0%MS;oP``2WN*yvB2R zmT?iwvy3#QViV87_jp3N|5hx6bZq*6g&7Fnq-;$8&fNXb{|GZe%K>IQy}{!Fk#KQc z0^_?ppN%qxb{F34P($w^_G)RyKTR{9Y5{{9^*_Um)Wt#k!XE%-IPU)pl;M4fY~Bkl zfL(rK@SYhkQ8lMo5qTf_celV%y^&*Qn9HqTOY;&rZ(1H_#JqiF^wxdI z@C7>VhnNqTLFF00ugg2rYHinv#%OtPF*m;CMxN1g-=r-qv}6X;$a&HRO~fv0U@_6@ z5DAU%ylgITnInE;&;80j`wTL|)`;W{L<0<#|+l_Z;AEGxTve zH=4QHvveymueY*jPs0snw2?>A=kCmj-ca!agS#-*j7G(y* zPi6*hC+B;t%W@;0!Z^~6zKEArn^lN0iO?96&&tcsty&mB(<~N$pmV^1M zT0n!_(0QKnKJ?%mUmF!>U-+7y7}|}#wV4x8=n=Z~aL#-d`vGlS%u?sJ9{nN9hW%cn z<&Ek{WcUcUeRjwk(YFJWSeB8z;t)b~t@!HlX@?^ANHOYi0!XK>c0cFwSz0f2?BPWDMOl0=$kvCpuaeQ5Wf zD?7n*cDhcrob7u&WGTpi;|xF4r>v#VKa*Mm1$&A^b$GvtCMvZ~q!;xDv%#8wy05(D z>>ngG=jgodI5VWCbzd_bXO8oCBo|#egR1XL{E&FPj&UsL3|graXU6T>x;yhDvq1feML8%IOrhy{?>G}&x}!&F#7H$!9!O!gXC+5#|(`- z$X5YP8|sM_WZq^*-TrWd9C}9enN`^d!_&9&nU&5)>8fEF^?Y^GWmY;n=qC#r@(#0u z;D{{fY`25ZYo@{Mm5fNtufD-zhA!M=iG1B#N0xtu==t)ukATC>68JKYN?Glsw=6T) z`u-;7Y8%*JsQF*H*;p2Q6FF?uZ% pq_xak|1|Ju)Ov=CI*HZPr?Ghipf#2wWs@@LoP7@0&&>n&{|m}FW5WOd literal 0 HcmV?d00001 diff --git a/src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.lzo b/src/bin/gpfdist/regress/data/gpfdist2/lineitem.tbl.lzo new file mode 100755 index 0000000000000000000000000000000000000000..365a1bbb19c5853e2333f6716463c9ae684dd6e4 GIT binary patch literal 13937 zcmXYY4P0C0**+(MmWF&4Xr&a|!}FYzlW)%VM@wyMX@yGd7qwJrmy`s8rVvP|!5wW{ zQ%yCkX-!?KsiLC8NjKc+MwL3;*eg@&a6`q38}6+e`m(*am%ZXv|NB4v|Ni{)D``uf zbI$YJ_jOz9B?^JiHjG~Ax znsYJS7>2Pfh8@FlB+kXMYW3E-+Pk*y!E5itcWvFXcjujI{hobW?ojSTYGLU&&uP*oGrYIrPcb) zcWlLP+q~rte9zYUeS7XycWkb$Q-j@jb8DM7i2M8@f1A(W*3{|`ufxN^ZuCEg+k-y* z=5=^gP&rjrC5d6(yo{cRp<7L?Z5G2A7pqJgFwR8_=z$m)$I!EwY0uW%@y)e+)XvuS zF0_^#@9bThQ|!e3(NL#99KLxqv@WOK+$>gBS%H;#H%;H%D#d4V{#MH}Q#Tjo=rL@m z;9^Ae&YgE|Regq>vLM?2up4gE?YZ~>KQS$DxYIjR^iZta%@+L z6v}a$^Qapdw3)OtslWM+W5)Ai@)B>Szqu=j?qI->ljo1>Se2nAbR6ipWb|T1#VyCE zvpK?CDJG$-j#VeH{?c6Z7OlZDV@o1iLzX3-{*EqxI1+XTc|&`+6`k@isNJ>o^q!0z;#lxNhN+|3v8n{-Q{P|(gclf2%LbST~@MVq6+%PO; zmbmbauGXgeg58T54Ic6~-IraPUB1+fxB7zJIzeTQ?4kp2FF|hvh7;XF#(erXZ+%G5 z(2l`8ZM(WMuwi)t3~mvxY%=- zdDl^mqQ>+dITu9_-WB#wOS5IhaLC`(iqg5YEgb3WYSIajs8} zyP@1%M@GRT59{OqiB4K%0(qVo=5P14yQfx8bmr+qmEmbs3}%-|D4G$zxS6;yl|xNo zS`@zsQZjuv-=O4-mnSebS@gU=WUI+rD9?0ibqM`r?$W>6hhmC6J%7=`*~*s*(;;UG z9>M6LQ+P&jvqzfgRJr{d9UFb$ z@{bDo^cnH!B}SpAu*8|jWJtdZJz}J_y)BF`sHmSUuBpmoqM!?P8iVVp?wZ`TUvr)gTSg$L-aGPqR5O}O1GV-7~36aI~dE_3WU1p z-<~RE*m={Xt!*Fb{doLz>?>_+XM0;~6Lo)?8$aA(mSZ0*MX-%vmFH-7{O?)W{&_w# z8a`=aGrp54mZ@9gyM5Wc?Lm|-{%8|A-jUm_c8xb^D=v>$_&<`=s~T3J-4g#NJL?+g zd1bx$zMBQIg9@3kR+RnJND-z(P&U#8eq@mZcHvJkJ#|ti1d^9Z1<#4z@MVoi$91nb z!-?0Yo&YadJE8`RQDimt?xpU-1&ZXz>bz%yj@{Z$ zTi()U%ak=>!VpBU>c2JU(9zi#A?~JdDyILNfuYBbr#0Nd+K`F5?f@PLhARf&IEr=p zK%GsCP9NQ<$E^M(k4JTY6B(9_lUWm0vMg>O>hPg87w#60T&2-J&V97_22n(#)N0bp zq8hAT%A=2PoGiJ8v^}X%)2^hRh%=CLRY`5Cmu6Vt;?j~@9nD!_kaca61=6g869|&H`r_)F)nfI zI0PuHDn)+11eF>^W;q1!;R|;Dt0^wC-@3}ie1S26PwYe3_xGcsG9cPEtr{;QWB8*c zMwLC=m!M6hC4qA*DHkdVw$Cw^%yAIoWa>%SOt7+}tN_;0#7ZR+pmNKP=^0KAZVMjM zpm2AK83kqDP+3d|k^pu%*^rgUaonu$2uF?4yOy}|aK!HoMgm}$0Dmr{i0L+PEk8ti z1d3S{C@7VZZ^Z2OdYh%uohj}AAw&_&)Z3D)$z0r;AT_@BuEUX^Dmu>EOM$r!7%K}h zDLdkCQs0)!apml*a^&zUaT#W4azdW*1+Mt^1v+hG9EPeO9(lFZsDK&@B`)GIEF;Mb zk!{(bu+8Xq?MV!=&7DSiGz(%aSJseSu7{{m* zn7mmI(NBu~5eSHC_iDzoHy2fXM6-mc z`==kVhIYW3YT_jaGd41;cP`p4S(`wytmz6Oo(T@^aO0LsdFy6t^E3#LI|89Lq3;NnS34?e%eLfb7V`|zCSu}%}iChhquhj<1}3zA#R z%IP0d&Z@MDpey@~utYeoVVQhm!U0FCVdGA#M$_ayW#qD_mhy-mD6F8k`GkDZC@)@1 z77%)C0_*KKYog`(J%n?o`c|F}u`EUs)p#-&g}HRuu0g}^_cAQy z|0kH)`>`t7N60$9{9L&{zZlFQ_#>`zdeq-oma(I*0 z+tk?}4#!RpjP!)N3?i6y6OE#l7dQmi>Bmz!_JS3th@VP|Fn0?)3B>fL;)M+X`(Igq zvw>yc)IQk12Yf+_XGICWVM2+-@I9hc=cF4$gyLJDhNS& z8RChk05Y_QI76%+UcbXODuR*QU}&RLSxN`+%K3b->7IXdwmwj%2=S+V$Fzz}b8ZnP zaRdz+ZS)YKLmvnKeF(8X_+~xLI5Jv56LXiz6qSouvJN!@$zR@tKlGv{+Lbg>wcF^# zA>zH;6&Epl2kfin)arkpZVHg|>$8vLLN zGQ`jhqIUyzqDXLoY_%n?k|#f-1UObrj?hGngDl=j1$Y?S37rSYt&XtI*KL$QT@AU5 z(C{pxbg7zzlZxcQ8}9yxFu_HOB++nLqc|dY0iPBTFuylUeZA6&cOKKTssI>~0Vwf` z<+wieAT>zEbp4sW(sajp4ad%uCmyoQ`H5bSHKGmt3^=VP*EYr8)#yY7Jzx_B%B)LK z?r{vlCpVwocVW3~y>}`~)?~34&E0$>obps<-QQn_w54WE+ zOW=JM3npB7JWun8cT1t8@k)E^-R#82N%O$RD2<#>CU7_9q%^3TK?k37q+Zvcw3B7~ z+E2nG>YxX|9 zHSDK;ZS^|z1=ipZq6Y6W^QT<`FcTQdh`eQ9iC5T{=kgPG4d54Aq>hJacGs$zN-A1I z5nBUBR2dNY1Go)EHOBz2tR&oUl5s_sLe<&3Ha$Fx(bIX+hYel9NNXtA>hBEWI@Acu zAal_FutfQ|Oeyu>6^PpgY^`s|sV6dqo7+{_9@d;$o@VigJnAz5R?OJV|Rf1Lm<}zJv+1I1aY)w7&X^KRaA>wjONxT`QSD$ z80Y|BtZ&IW(ZvyY-Mci{ zjbNDun;$?4(5m5RTH(m~9og;n1q{{vPq#~C-GS$v46jY& z#8U>6u8WpvC6SjA_bEH?pe?I-J|2ggZtKJ544$9Rb6cXHlO+~QhJXS8g9dKsYA$ML zWrlYX`46WIw#jyoLnKe63|IWT?@3Ei=|5rb$;WPLCEW$eu{{WG1OL`Yw1Rd+Ooi+x zP?taWRgeC!1{oB`=(-h)Ls-P$6tS$poy2TatC0cE3*z_RW;lsdcP6~HoIEgp4;-zD z^Mu|))H}w!7xuQn*gcdF<~DhCh#`q6s6l(Zgve_BBgCZLE|3g4{5X+Mev5LW?}5tU z$2&(`TEL*sTAe>5Pz#T@m>5MO^T86-q4O--Fg{^}0h*e55RMXrT$eV1k==0X!<>7O z@oou%MaD9{nP|WtH^{UK8l{KkWt!B7&tDL1*Q|_BN3_x$88}T0*NclANu0Hf97 zvx8bm9=LcWW~e0eu)lR?4e+^QCOAP>9+cXs_#^kf;>J53(hI049S@CYWrP@RCAK#v zQxjaD^&cMAT3f_|+1-_k`SE?$sRJPK2sOWMc0cCUNer-gInb~V+=>|_HOv1#zIzw@Xy z?WBV3kzXQcLTXS8>iKzC90b+1T7^N3NJ$&gF8jb*%k(o@Jw;M1;vLMc6`<^1Yu(w= zN!#S@frrnVlm|OSINGR{SOo5TYCqKD_7UxkI@!t}?&s`Z4B%~$!LoSzPxQ?Ld<36w4S_IIv z%uU;m9~W&CMIhov*VrvG9ln#U+QRRrSSn z>=!035GHn2p{6ZL_F;|(ptl?JGbG<(9eBjn_w*5?0InQjCjB0;^~$nwKiL z`ZgEw6Lia|=N%>q^qg1!I}i1ug3P(O#JllYs<_-&qkaxw zYNsPu#I5FllA6krVwj{M-t`v;5-2Opra|DZL&4yH3oYO;9*$SYnsX_$t$7;saT%nMziLARE0XJVQ<>>T!u+3F=cV{+Fbu4b_2}_KtYDw=CeZU} z=wWEyID8&{tk6#9>8Wu`-c$Z|L;L-~xPi>p6P^Yw%OUxlk6YtmJ6R!;X({#zF}>ro zft7MMS{JjGYwXRf=S-+#S)(BE42ph{v+KjujXlZLgr}*`WLaJzfRv>&cNx4vz==1% zdBP1xb|V!Ke)tNnK6;$t#qsjQbJW0@m~rS>0_*!tvYd5Z1SLPBtb_IfBi_>aq*35i zaQz5KxG;fLl#%z+Pf+#8U;-cV4V1xRXIjG!z4_w0HXjb~yMyn7AH5pZF|8{YpzLJe zjmt=1LV6+RsV5e3>Ix=;73*18(IBS3fj#1&hLxAj4P%k8%SA?i0U8Nw^(6`~{*f8c z^q{}lFLPjVecYMYPW2Np6R4u0+Tpq6J8;@ev&nLwAC8SsWp=7x*0%>kh^B|}sH@Y9 z41ph-IR)Z}TlOJDBkO-cjQo@u+wFN44D5h8A!^hQrkjGlsq}Hyz7-^~6+`F5TCG31 zsS}C3IVWkpFvq=8glE7@=Y7tVbd-|Qjt$_dec+1eylpY#0=oALx7>~3z(rcGMzMt|J zN`V7I4=445J!m6KmSsy}eGPSzb6e^1#7X$o%Q|obbt&{du+W7lmc$U(FA?KgsN$7y zx(f{4nrVR9XVSHiZ^(!pB)MLz<0UNmiXK(vpF_pY0qj;TvpO+ACAApxdlXWr60M&M+!_-^P~XD{X~`yS>%v8&$;kD@Nak-KT&4NX_8KN2KIKe8ojsNC9%Mh05H zOWg8%g`r8?6rFHTH2@S}CU3aG)F*aZ4$YLjK6jOh-D0WwUfc)^8#xtBZA48A%aBu( z&;#rXJj{8S$l)xRcLvuut&UU&X!s>4YlJ251Hxs=*>oITqiY=xr#GOSV&$>_9se~s z_|RNIjJe7H#ziN-*GY%v-`&5W_VH-LF4@>0tH4QGY_3KGnU;h4(7iFqrv#v zzJ7En&i+SZPB7`Sl(^pd2tQ*$veQgE$Xr5CBr4==GkPK_bvKfoQXPnvH?Xq?y2=qL zx8Hc|A4!~`Z`YvI(=i||yl{q5fyKh}l=+clGnLt!=#2cl+dr!qz&dOn#=U|n^m- z(+N!E8Pp#ii20oj-y<4^Kw>XBu`z4p>RqY*BezfXYkmHv_U*!rI`AkM*E_Y$gu zifQ58eSte*=I)FhPNL4JX;Sv=yJU6t@6CPy4}8`{sP{f?;00KH0k}VqSgP!m^a-lu z^+7}O@37tvvKvQCmYD~JLqvmrVg7@9C)K|V^LsmkL^62F#K4lDfbaf}=R`~NLafF% zu?Y-L!A*SJ>$raO+`ZkzYKT<{|zszgZAZuj|F*h2!z25-p1_8eTB{k2nM>?8XnOo`54eqAQZ z-{Sy7uw?pyOP_aJ{#uLMddsfr8DjQ!SP_m<~X&~x5(i(|)$C5bebgoeMY0v|3DsiY0 zg;@SWn8cvu@5CCQPuiP2+&8s-Lb@^m*SNrCuRFtXeFFE6r!Uoc5h z><0k~V-;2;0|TihYSW>Fo_NqLgK0Q)1u6W=BFkchwQ+^Dx@ zk#UsDqJl8N^_Rn5E^X@Z!GjxUu4g@vJ&_t)4IOyBusCP#jnctj^brq#CtGZ^uK?&I%pYS=-8goHfr~VnvsZ-Tp>bjOOACEmgN!G!}dl$HK~2gEWnBXB&v8M^r78r z{BEL$n!7z=B0MLE(YImLmR!U3|DU9vZ1_X(?b>h zIyOB82VVtr$T5$d^|uxFmw0@Zac8m5oH0ZvKx2R-ywuu3XO~%^G@V;m289a zM+OaW^gg1Y0p;-E)!zNq8FUZUauwFEvNzoF_jkQPGQm>2)2*cc8#kPz`%OUpWnp?1v_B~EWGb9A z*)}cn7&{mxW+VtPI16eP=&Isjr)%oB>061uSE=Y@2{VW{0mE*oFp7horU?_Q``^i< zi1EX!&Cw~cDg=WTS4rK%+M;Se%C8rdflS>ABcl|?R)fSR6Pj$j zn<<=FCiTR%_>rz~1cx(^_ZUFX1r{cozTjAa*!7epP-zcnsp>p9{w@r>1Mml(`M(>o zgUC);w%IzICIdB>kTZ7ix_I^y19KNr4}WHrLQ3M}uwkbqvCU(GRhAx;S(lq1GLL*Yu22gsbsb;f zDs_M0X26fX;Z@l25eM+^WuJsrD>bq+Z4L~9afh!WZAQV7Doj1852jukM(NWmbn8ovAPnq>Rn&3!d#3n%SZ+^AfSNJq7@+iPM34MWy!zHz2#ilfYp;M~R_l5&6d@&0c3^(7RlaN)l}*?ZvD*N~Q9^;+gRyRd~L zFLj{GdzL^G8!#1PbI(&{8nWas7M^+RukGQq@yN79uG%UTmXJN~rQf6G^u)kjGtHLk zWBq1!@}0pap>v`?0D3nUj~AdWP*DpB8B}cK;Xp@nU-x?2K8}H@6qs?v3}9$K9BPEw z`|YXK#Fz_48mNIh?GZ1Md%Y<`VMB%5lZVE21X>Zr6?i^OS?nWTGqxNlx${4wK104i z$}(>+3aupgz5Qk2*zTY|3}(}F2i`IZg=6c9*e|uRz=&=-Q=VK;;RR`VXdqMm!Xuok zFagQ>IvBMgqqV(@oO}r0&H$H5~GKkmq7!0TNo-gp0jhE;EY+^03jexmsm2RBM!%;kz{(v8ydU-$ultXjPEK;Sdh=i z_iP8lm5x`M+KaljiJ<>=@YQ#iqUWn+6sCm!J?G$xOefLf=))50hHjS|j7{~vuFj&I zQVklR>e4kSjKqS3BXvnDD@aQg65CQaW=)5QPPG%lK90QpR%+B#@6*; zyM$Q&v#6Om+#{fF;+t44k_k~0GFh8gL)3W^n7Lqf) zIQ9`*9waTvtMNN1q99u~^lr>J{OQfUd&*dRy$061V2%QuN2#Hu1X6<<$+QW6@TdSM zA4Ovyk^~0oEY1q*hB0Xj!3$1?T&TK$yx+#B?YrXRKSB)=UlW!ka35@^ zg9GG*Ye)yHIl|1Oj}pm!_UkJMTUEI5%Z+}orAGz_H|fA~26?v(GM~|)nZhAAKJ`Y@ zZcnVT0x7bb+H#0gy~dK^*)e(xp=yQYAC;gU|M;ad-13&@*_vn<T{myfGMh1;C$o>R%h%#?q?6S30()@TNUNH1Y0$cgn z-aVG7;H+e$3#oV>@vGZ)fz`q3!jS^hLolp>YB&B=GGd>+AN2hTBmivOZ~FwTq{8&p zQ0b|M@VVbVRe(B`-Zu-!*C2gX5=Dj_@n;TGv*hqCNn{P#x{(bxumXj#Jlku6r}s%k>3lRP!6B(} zS1N29;R9c$K2Er(`or1v_VbQC(>2t+dL&Dg{L#{|E6>;7(&9(sr{Jv%GMwED-s@8s zOZL;alk$0B&m_@k0oM1)$#aP4E^S=36RDJ_tTKeK^>9!K8pR0jB?i z-5j7STvn!d{+S6=qb*2|n+hke;fMQ|k2SpR?2Lf1ZHe;Y3Vl;?jyG~Z#@_F5e9_Zln>Pff@mv=vlY`b*=6qIph(OkGwXi~pK@mKs~0 zzJ(f(tsDr!qeDgGwHTThWkAhu;807T^CmA7WszL` z6YSkgT|2=+LbhBGg^6{@mmasnejW^^i4%Nl8&Q2$P!D@dU~GTyIEM5#dd1YrtdXjz zf(zdzLKa9{Dc@?Ke@QsGDol-Szh=^bQ-4;FkIEFz6S=6wBSS^V&L*0v+&#XgXeUg& zfO=m9fP=r5)uWXIM$hxbIi1&AkTu}*f<*^xtbN+ZNC1!Ku_!WRkrhU|Qg=}!`l6~8 zp7&t>yC^{-KSFq6F0gXPUES8vPvF?zxfWaB+k}S$)fRYO=FCg=51g!Ht#Jm7#N!{s zz5~Qq1n7S*Uf&NI`o|)#l*`47t;iu_5lSLa98WzAuG@;_cI05E9Yv89R2qXhKd+4E zfovtH|Jn7kfl_PLG}5ovSOhiLgl1x7natLq=&%+i41Fl+`oBh&I+LwS1VHf*h=mQ4 zIYkp(Y7r?^K<_Bpg`_Q-7VZ7A?U8k~v|_-TT|*@65_VVrDx^ydZdg&j#z7y%!foxk zLfu|LD(ca}@4hI=?23KcBwr6zvXsc$V9!3dU>kfB<%U@|Q8M)8q)e`8Z%(0_eg~+k z^5mnw5e?0#-)6%0Io@_XW|GcKE$b_G{k8tj1N!sK(uk+(8#B9=Hg8XWu!YiayT)1wmW#x|LbC+J9&Vb+n3b$ztFEw-;dJB zSz{k4hkXx1`3}ou)Ji-vXoQ}QLY_yJ35~`|$i;m=c9KHIdSefDR;x^>ksMN2oO;h% zH9+9-G8*H#ILX5yH0vtU!iu`P;|j%dV0a&xd zrofHwc@n413g|j06cMK|G^oUbWw+AgoJe{fj~yJZv=rN*x%0?9K?~uCn?RI=^iK*KN-Sk>Cr;Tv z#YB6K!}n}z4V?N5H=(^Hv;3B_P1H-Jof#uoxCA_lM@Cps<1flT(|;?KeG00O@OYEA z5^>bcJ0oqeHPo&J>qf)juJ9ED5ALp()WWI`OpuZJIe&==w%IRg6%hOUtev)$W#OpT zGJk1dd#sz#Q92gv>M9K8hXNRf95u4aosuZzN2Ko|>Lp8zBO30Gp11Flnus`O&7QRS zR@w&6oi{Q?`4z=4RZ2xOZGU_bO}kS)r-%Ur@i1l@ln`_{M+Uq0h3$bogH0hznYXCKZB8VB6#qSLUb!m+ReN#rJH++q=x@lGTYLY|5f;Hed`#g? zZYYegXs)n0**GlWn5=+0KQnCwy@D;2XVFoi*0W^{9JmCAk>*V!XNhRMH~p!71S{$- zr|_llx+Pi0mOX#sMQ_lo3Dr@_M~T4ctPx&4Ai{V5Lt^BWLxJfxNoNI|A0%>bgE{Ma znvg;=a^drR{^UPbMEo{c z*`P)K!`bcOfqiC8nLagKI5=3;cPIS(q!9VCI2p;TqlQ-#GtVZKzJm0ZH#fKb@%TFE z>9S4zm{`~k>LhE-``kS`mW^Bj4JQi!`ZBX3_=CX)s=vP926LZ}*Y|;C{HUb>)q`b2Xf5&(;pqILB&XbzH74FI==+SN)`|IR32dhL%h? z!9*sx91PAB$vfaDugg5YqO3XPruuIooZGYiLivT0@u1za`^1{C^PvK7(2_HG(jdY9 ze}PRWwCL9Jvp4IN>Iht9pkYhNhf{0=rV?er9;RuLsiM%VWO zin@i4R1Fxcw|rfRM(GC(MLU|=!pSFGH~xo;8<9&A|FRETxC>{3JAt%<_Q$?0#m&z?Rt6#jUm}Ktu`b=B#bJyePc{uJChU z*T)%sLZhq++#2cCVl#N}oGf1Vfb^tb4iqhBUNU;h`|j0@cW literal 0 HcmV?d00001 diff --git a/src/bin/gpfdist/regress/input/gpfdist2_lzo.source b/src/bin/gpfdist/regress/input/gpfdist2_lzo.source new file mode 100644 index 00000000000..491494e4471 --- /dev/null +++ b/src/bin/gpfdist/regress/input/gpfdist2_lzo.source @@ -0,0 +1,184 @@ +-- +-- GPFDIST test cases set 2 - LZO compression support (read-only external tables) +-- This test suite verifies that gpfdist can read LZO-compressed files created by lzop tool +-- Note: Writable external tables with LZO compression are NOT yet implemented +-- +drop table if exists REG_REGION; +set optimizer_print_missing_stats = off; +CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY); +-- start_ignore +-- end_ignore +-- -------------------------------------- +-- gpfdist protocol with LZO compression +-- -------------------------------------- +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start; +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_stop; +CREATE EXTERNAL WEB TABLE gpfdist2_start (x text) +execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data /dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); + +CREATE EXTERNAL WEB TABLE gpfdist2_stop (x text) +execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); + +-- start_ignore +select * from gpfdist2_stop; +select * from gpfdist2_start; +-- end_ignore + +-- test 1: using a small LZO-compressed file (created by lzop tool) + +CREATE EXTERNAL TABLE ext_lineitem_lzo ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_read (like ext_lineitem_lzo); +SELECT count(*) FROM ext_lineitem_lzo; +INSERT INTO lineitem_lzo_read SELECT * FROM ext_lineitem_lzo; +DROP EXTERNAL TABLE ext_lineitem_lzo; +DROP TABLE lineitem_lzo_read; + +-- test 2: use a bigger LZO-compressed file + +CREATE EXTERNAL TABLE ext_lineitem_lzo_big ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.long.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_big_read (like ext_lineitem_lzo_big); +SELECT count(*) FROM ext_lineitem_lzo_big; +INSERT INTO lineitem_lzo_big_read SELECT * FROM ext_lineitem_lzo_big; +DROP EXTERNAL TABLE ext_lineitem_lzo_big; +DROP TABLE lineitem_lzo_big_read; + +-- test 3: verify data integrity - compare counts between original and LZO-compressed files + +CREATE EXTERNAL TABLE ext_lineitem_orig ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +SELECT count(*) FROM ext_lineitem_orig; +DROP EXTERNAL TABLE ext_lineitem_orig; + +-- start_ignore +select * from gpfdist2_stop; +-- end_ignore + +-- test 4: test reading LZO file with larger buffer size (1.2MB) +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start; + +CREATE EXTERNAL WEB TABLE gpfdist2_start (x text) +execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data -m 1200000 /dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); + +-- start_ignore +select * from gpfdist2_start; +-- end_ignore + +CREATE EXTERNAL TABLE ext_lineitem_lzo_big2 ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.long.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_big_read2 (like ext_lineitem_lzo_big2); +SELECT count(*) FROM ext_lineitem_lzo_big2; +INSERT INTO lineitem_lzo_big_read2 SELECT * FROM ext_lineitem_lzo_big2; +DROP EXTERNAL TABLE ext_lineitem_lzo_big2; +DROP TABLE lineitem_lzo_big_read2; + +-- start_ignore +select * from gpfdist2_stop; +-- end_ignore diff --git a/src/bin/gpfdist/regress/output/gpfdist2_lzo.source b/src/bin/gpfdist/regress/output/gpfdist2_lzo.source new file mode 100644 index 00000000000..49052596b68 --- /dev/null +++ b/src/bin/gpfdist/regress/output/gpfdist2_lzo.source @@ -0,0 +1,218 @@ +-- +-- GPFDIST test cases set 2 - LZO compression support (read-only external tables) +-- This test suite verifies that gpfdist can read LZO-compressed files created by lzop tool +-- Note: Writable external tables with LZO compression are NOT yet implemented +-- +drop table if exists REG_REGION; +set optimizer_print_missing_stats = off; +CREATE TABLE REG_REGION (R_REGIONKEY INT, R_NAME CHAR(25), R_COMMENT VARCHAR(152)) DISTRIBUTED BY (R_REGIONKEY); +-- start_ignore +-- end_ignore +-- -------------------------------------- +-- gpfdist protocol with LZO compression +-- -------------------------------------- +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start; +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_stop; +CREATE EXTERNAL WEB TABLE gpfdist2_start (x text) +execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data /dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); +CREATE EXTERNAL WEB TABLE gpfdist2_stop (x text) +execute E'(ps -A -o pid,comm |grep [g]pfdist |grep -v postgres: |awk \'{print $1;}\' |xargs kill) > /dev/null 2>&1; echo "stopping..."' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); +-- start_ignore +select * from gpfdist2_stop; + x +------------- + stopping... +(1 row) + +select * from gpfdist2_start; + x +------------- + starting... +(1 row) + +-- end_ignore +-- test 1: using a small LZO-compressed file (created by lzop tool) +CREATE EXTERNAL TABLE ext_lineitem_lzo ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_read (like ext_lineitem_lzo); +NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table +SELECT count(*) FROM ext_lineitem_lzo; + count +------- + 256 +(1 row) + +INSERT INTO lineitem_lzo_read SELECT * FROM ext_lineitem_lzo; +DROP EXTERNAL TABLE ext_lineitem_lzo; +DROP TABLE lineitem_lzo_read; +-- test 2: use a bigger LZO-compressed file +CREATE EXTERNAL TABLE ext_lineitem_lzo_big ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.long.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_big_read (like ext_lineitem_lzo_big); +NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table +SELECT count(*) FROM ext_lineitem_lzo_big; + count +-------- + 100000 +(1 row) + +INSERT INTO lineitem_lzo_big_read SELECT * FROM ext_lineitem_lzo_big; +DROP EXTERNAL TABLE ext_lineitem_lzo_big; +DROP TABLE lineitem_lzo_big_read; +-- test 3: verify data integrity - compare counts between original and LZO-compressed files +CREATE EXTERNAL TABLE ext_lineitem_orig ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +SELECT count(*) FROM ext_lineitem_orig; + count +------- + 256 +(1 row) + +DROP EXTERNAL TABLE ext_lineitem_orig; +-- start_ignore +select * from gpfdist2_stop; + x +------------- + stopping... +(1 row) + +-- end_ignore +-- test 4: test reading LZO file with larger buffer size (1.2MB) +DROP EXTERNAL WEB TABLE IF EXISTS gpfdist2_start; +CREATE EXTERNAL WEB TABLE gpfdist2_start (x text) +execute E'((@bindir@/gpfdist -p 7070 -d @abs_srcdir@/data -m 1200000 /dev/null 2>&1 &); for i in `seq 1 30`; do curl 127.0.0.1:7070 >/dev/null 2>&1 && break; sleep 1; done; echo "starting...") ' +on SEGMENT 0 +FORMAT 'text' (delimiter '|'); +-- start_ignore +select * from gpfdist2_start; + x +------------- + starting... +(1 row) + +-- end_ignore +CREATE EXTERNAL TABLE ext_lineitem_lzo_big2 ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.long.lzo' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE TABLE lineitem_lzo_big_read2 (like ext_lineitem_lzo_big2); +NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table +SELECT count(*) FROM ext_lineitem_lzo_big2; + count +-------- + 100000 +(1 row) + +INSERT INTO lineitem_lzo_big_read2 SELECT * FROM ext_lineitem_lzo_big2; +DROP EXTERNAL TABLE ext_lineitem_lzo_big2; +DROP TABLE lineitem_lzo_big_read2; +-- start_ignore +select * from gpfdist2_stop; + x +------------- + stopping... +(1 row) + +-- end_ignore