From e83ff926df003eb03be2ff4f2f7c6a09262791da Mon Sep 17 00:00:00 2001 From: ag-ramachandran Date: Tue, 24 Mar 2026 20:25:17 +0530 Subject: [PATCH] *Allow transformation functions for CSV and SCSV formats as well --- .../kusto/ingest/ingestion_properties.py | 2 +- .../tests/test_ingestion_properties.py | 20 +++++++++++++++++++ uv.lock | 8 ++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py b/azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py index 99b0b09e..388dedbb 100644 --- a/azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py +++ b/azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py @@ -72,7 +72,7 @@ class ColumnMapping: FIELD_NAME = "Field" NEEDED_PROPERTIES: Dict[IngestionMappingKind, List[str]] = { - IngestionMappingKind.CSV: [ORDINAL, CONST_VALUE], + IngestionMappingKind.CSV: [ORDINAL, CONST_VALUE, TRANSFORMATION_METHOD], IngestionMappingKind.JSON: [PATH, CONST_VALUE, TRANSFORMATION_METHOD], IngestionMappingKind.AVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], IngestionMappingKind.APACHEAVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], diff --git a/azure-kusto-ingest/tests/test_ingestion_properties.py b/azure-kusto-ingest/tests/test_ingestion_properties.py index d0251e61..b0b3837a 100644 --- a/azure-kusto-ingest/tests/test_ingestion_properties.py +++ b/azure-kusto-ingest/tests/test_ingestion_properties.py @@ -90,3 +90,23 @@ def test_with_constant_value(): data_format=DataFormat.PARQUET, ingestion_mapping_kind=IngestionMappingKind.PARQUET, ) + + +@pytest.mark.parametrize( + "transform,data_format,mapping_kind", + [ + (TransformationMethod.SOURCE_LOCATION, DataFormat.CSV, IngestionMappingKind.CSV), + (TransformationMethod.SOURCE_LINE_NUMBER, DataFormat.CSV, IngestionMappingKind.CSV), + (TransformationMethod.SOURCE_LOCATION, DataFormat.TSV, IngestionMappingKind.CSV), + (TransformationMethod.SOURCE_LOCATION, DataFormat.SCSV, IngestionMappingKind.CSV), + ], +) +def test_csv_mapping_with_transform_only(transform, data_format, mapping_kind): + """Transform-only CSV column mappings should be valid without Ordinal or ConstValue. See #627.""" + IngestionProperties( + database="database", + table="table", + column_mappings=[ColumnMapping("test", "string", transform=transform)], + data_format=data_format, + ingestion_mapping_kind=mapping_kind, + ) diff --git a/uv.lock b/uv.lock index 47bbda24..a976d6f1 100644 --- a/uv.lock +++ b/uv.lock @@ -229,7 +229,7 @@ wheels = [ [[package]] name = "azure-kusto-data" -version = "6.0.1" +version = "6.0.2" source = { editable = "azure-kusto-data" } dependencies = [ { name = "azure-core" }, @@ -265,7 +265,7 @@ provides-extras = ["pandas", "aio"] [[package]] name = "azure-kusto-ingest" -version = "6.0.1" +version = "6.0.2" source = { editable = "azure-kusto-ingest" } dependencies = [ { name = "azure-kusto-data" }, @@ -291,7 +291,7 @@ provides-extras = ["pandas", "aio"] [[package]] name = "azure-kusto-python" -version = "6.0.1" +version = "6.0.2" source = { virtual = "." } dependencies = [ { name = "azure-kusto-data" }, @@ -1770,7 +1770,7 @@ wheels = [ [[package]] name = "quick-start" -version = "6.0.1" +version = "6.0.2" source = { virtual = "quick-start" } dependencies = [ { name = "azure-core-tracing-opentelemetry" },