Skip to content

Add schema-serde streaming JSON/XML deserializers, ec2Query, Codec, and ClientProtocol #3910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pulimsr wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add schema-serde streaming JSON/XML deserializers, ec2Query, Codec, and ClientProtocol #3910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changelog/feature-schema-serde-deserializers.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "feature",
"category": "aws-cpp-sdk-core",
"contributor": "pulimsr",
"description": "Add schema-serde buffer-based JSON/XML deserializers, Query serializer (awsQuery/ec2Query), and Codec/ClientProtocol"
}
41 changes: 16 additions & 25 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,35 +12,26 @@ class SMITHY_API CborShapeDeserializer final : public ShapeDeserializer {
CborShapeDeserializer(const unsigned char* data, size_t length);
~CborShapeDeserializer();

Aws::Crt::Optional<bool> ReadBoolean() override;
Aws::Crt::Optional<int> ReadInteger() override;
Aws::Crt::Optional<int64_t> ReadLong() override;
Aws::Crt::Optional<float> ReadFloat() override;
Aws::Crt::Optional<double> ReadDouble() override;
Aws::Crt::Optional<Aws::String> ReadString() override;
Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp() override;
Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob() override;
Aws::Crt::Optional<int> ReadEnum() override;

size_t BeginStruct() override;
void EndStruct() override;

size_t BeginList() override;
void EndList() override;

size_t BeginMap() override;
void EndMap() override;

Aws::Crt::Optional<Aws::String> ReadKey() override;
bool IsBreak() override;
void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) override;
void ReadList(const Schema& schema, const ListElementConsumer& consumer) override;
void ReadMap(const Schema& schema, const MapEntryConsumer& consumer) override;

Aws::Crt::Optional<bool> ReadBoolean(const Schema& schema) override;
Aws::Crt::Optional<int> ReadInteger(const Schema& schema) override;
Aws::Crt::Optional<int64_t> ReadLong(const Schema& schema) override;
Aws::Crt::Optional<float> ReadFloat(const Schema& schema) override;
Aws::Crt::Optional<double> ReadDouble(const Schema& schema) override;
Aws::Crt::Optional<Aws::String> ReadString(const Schema& schema) override;
Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp(const Schema& schema) override;
Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob(const Schema& schema) override;
Aws::Crt::Optional<int> ReadEnum(const Schema& schema) override;

bool IsNull() override;
void ReadNull() override;
void SkipValue() override;

private:
class Impl;
Aws::UniquePtr<Impl> m_impl;
};

} // namespace schema
} // namespace smithy
}
}
99 changes: 99 additions & 0 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#pragma once

#include <aws/core/utils/memory/AWSMemory.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <smithy/Smithy_EXPORTS.h>
#include <smithy/client/schema/Codec.h>

namespace smithy {
namespace schema {

class Schema;
class SerializableStruct;
class ShapeDeserializer;

class SMITHY_API ClientProtocol {
public:
using SerializerOutcome = Codec::SerializerOutcome;

virtual ~ClientProtocol() = default;

virtual Aws::String GetProtocolId() const = 0;
virtual Aws::String GetContentType() const = 0;
virtual SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const = 0;
virtual Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const = 0;
};

class SMITHY_API RestJsonProtocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;

private:
JsonCodec m_codec;
};

class SMITHY_API AwsJson10Protocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;

private:
JsonCodec m_codec;
};

class SMITHY_API AwsJson11Protocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;

private:
JsonCodec m_codec;
};

class SMITHY_API RpcV2CborProtocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;

private:
CborCodec m_codec;
};

class SMITHY_API RestXmlProtocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;

private:
XmlCodec m_codec;
};

class SMITHY_API AwsQueryProtocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;
};

class SMITHY_API Ec2QueryProtocol final : public ClientProtocol {
public:
Aws::String GetProtocolId() const override;
Aws::String GetContentType() const override;
SerializerOutcome SerializeInput(const Schema& schema, const SerializableStruct& input) const override;
Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(const unsigned char* data, size_t length) const override;
};

}
}
47 changes: 47 additions & 0 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#pragma once

#include <aws/core/client/AWSError.h>
#include <aws/core/client/CoreErrors.h>
#include <aws/core/utils/Outcome.h>
#include <aws/core/utils/memory/AWSMemory.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <smithy/Smithy_EXPORTS.h>

namespace smithy {
namespace schema {

class Schema;
class SerializableStruct;
class ShapeDeserializer;

class SMITHY_API Codec {
public:
using SerializerOutcome = Aws::Utils::Outcome<Aws::String, Aws::Client::AWSError<Aws::Client::CoreErrors>>;

virtual ~Codec() = default;

virtual SerializerOutcome Serialize(const Schema& schema, const SerializableStruct& shape) const = 0;

virtual Aws::UniquePtr<ShapeDeserializer> CreateDeserializer(const unsigned char* data, size_t length) const = 0;
};

class SMITHY_API JsonCodec final : public Codec {
public:
SerializerOutcome Serialize(const Schema& schema, const SerializableStruct& shape) const override;
Aws::UniquePtr<ShapeDeserializer> CreateDeserializer(const unsigned char* data, size_t length) const override;

@sbiscigl sbiscigl Sep 1, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

summarizing from offline:

ok i think CreateDeserializer is where where we have the disconnect and where it will come together for us with "how do we actually create a object T". so looking at the smithy java codec. we're actually borrowing a little bit more than we need. and we can actually walk this back.

so Serialize looks good, lets park that lets only talk about deserialize

in the java impl the have one called deserializeShape that is closer to our needs and answers the second question which is "how do we construct T given a stream of characters".

javas looks like

public <T extends SerializableShape> T deserializeShape(byte[] source, ShapeBuilder<T> builder)

ours should end up looking like

void deserializeShape(const unsigned char* data, 
  size_t length,
  std::function<void (const Schema& schema, const StructMemberConsumer& consumer)> consumer)

this is so that during coden class would implement those struct member consumers to construct self

};

class SMITHY_API XmlCodec final : public Codec {
public:
SerializerOutcome Serialize(const Schema& schema, const SerializableStruct& shape) const override;
Aws::UniquePtr<ShapeDeserializer> CreateDeserializer(const unsigned char* data, size_t length) const override;
};

class SMITHY_API CborCodec final : public Codec {
public:
SerializerOutcome Serialize(const Schema& schema, const SerializableStruct& shape) const override;
Aws::UniquePtr<ShapeDeserializer> CreateDeserializer(const unsigned char* data, size_t length) const override;
};

}
}
37 changes: 37 additions & 0 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#pragma once

#include <aws/core/utils/memory/AWSMemory.h>
#include <smithy/Smithy_EXPORTS.h>
#include <smithy/client/schema/ShapeDeserializer.h>

namespace smithy {
namespace schema {

class SMITHY_API JsonShapeDeserializer final : public ShapeDeserializer {
public:
JsonShapeDeserializer(const unsigned char* data, size_t length);
~JsonShapeDeserializer();

void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) override;
void ReadList(const Schema& schema, const ListElementConsumer& consumer) override;
void ReadMap(const Schema& schema, const MapEntryConsumer& consumer) override;

Aws::Crt::Optional<bool> ReadBoolean(const Schema& schema) override;
Aws::Crt::Optional<int> ReadInteger(const Schema& schema) override;
Aws::Crt::Optional<int64_t> ReadLong(const Schema& schema) override;
Aws::Crt::Optional<float> ReadFloat(const Schema& schema) override;
Aws::Crt::Optional<double> ReadDouble(const Schema& schema) override;
Aws::Crt::Optional<Aws::String> ReadString(const Schema& schema) override;
Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp(const Schema& schema) override;
Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob(const Schema& schema) override;
Aws::Crt::Optional<int> ReadEnum(const Schema& schema) override;

bool IsNull() override;

private:
class Impl;
Aws::UniquePtr<Impl> m_impl;
};

}
}
50 changes: 50 additions & 0 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/QueryShapeSerializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
#pragma once

#include <aws/core/client/AWSError.h>
#include <aws/core/utils/memory/AWSMemory.h>
#include <smithy/Smithy_EXPORTS.h>
#include <smithy/client/schema/ShapeSerializer.h>

#include <functional>

namespace smithy {
namespace schema {

class SMITHY_API QueryShapeSerializer final : public ShapeSerializer {
public:
using SerializerOutcome = Aws::Utils::Outcome<Aws::String, Aws::Client::AWSError<Aws::Client::CoreErrors>>;

enum class Flavor { AwsQuery, Ec2Query };

QueryShapeSerializer();
explicit QueryShapeSerializer(Flavor flavor);
~QueryShapeSerializer();

void WriteStruct(const Schema& schema, const SerializableStruct& value) override;
void WriteList(const Schema& schema, size_t size, const std::function<void(ShapeSerializer&)>& consumer) override;
void WriteMap(const Schema& schema, size_t size, const std::function<void(MapSerializer&)>& consumer) override;

void WriteBoolean(const Schema& schema, bool value) override;
void WriteInteger(const Schema& schema, int value) override;
void WriteLong(const Schema& schema, int64_t value) override;
void WriteFloat(const Schema& schema, float value) override;
void WriteDouble(const Schema& schema, double value) override;
void WriteString(const Schema& schema, const Aws::String& value) override;
void WriteTimestamp(const Schema& schema, const Aws::Utils::DateTime& value) override;
void WriteBlob(const Schema& schema, const Aws::Utils::ByteBuffer& value) override;
void WriteEnum(const Schema& schema, int value) override;
void WriteNull(const Schema& schema) override;

SerializerOutcome GetPayload();

private:
class Impl;
Aws::UniquePtr<Impl> m_impl;
};

}
}
16 changes: 14 additions & 2 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
#pragma once

#include <aws/core/utils/memory/stl/AWSString.h>
#include <smithy/client/schema/Trait.h>
#include <smithy/client/schema/TraitKey.h>

Expand All @@ -22,7 +23,18 @@ class TimestampFormatTrait : public Trait {
Format m_format;
};

class Ec2QueryNameTrait : public Trait {
public:
explicit Ec2QueryNameTrait(const Aws::String& value) : m_value(value) {}
const Aws::String& GetValue() const { return m_value; }
static const TraitKey<Ec2QueryNameTrait>& KEY() { return TraitKey<Ec2QueryNameTrait>::Instance(); }

private:
Aws::String m_value;
};

extern template class TraitKey<TimestampFormatTrait>;
extern template class TraitKey<Ec2QueryNameTrait>;

} // namespace schema
} // namespace smithy
}
}
41 changes: 20 additions & 21 deletions src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/crt/Optional.h>
#include <smithy/Smithy_EXPORTS.h>
#include <smithy/client/schema/Schema.h>

#include <cstdint>
#include <functional>

namespace smithy {
namespace schema {
Expand All @@ -15,31 +17,28 @@ class SMITHY_API ShapeDeserializer {
public:
virtual ~ShapeDeserializer() = default;

virtual Aws::Crt::Optional<bool> ReadBoolean() = 0;
virtual Aws::Crt::Optional<int> ReadInteger() = 0;
virtual Aws::Crt::Optional<int64_t> ReadLong() = 0;
virtual Aws::Crt::Optional<float> ReadFloat() = 0;
virtual Aws::Crt::Optional<double> ReadDouble() = 0;
virtual Aws::Crt::Optional<Aws::String> ReadString() = 0;
virtual Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp() = 0;
virtual Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob() = 0;
virtual Aws::Crt::Optional<int> ReadEnum() = 0;
using StructMemberConsumer = std::function<void(const Schema& memberSchema, ShapeDeserializer& deserializer)>;

virtual size_t BeginStruct() = 0;
virtual void EndStruct() = 0;
using ListElementConsumer = std::function<void(ShapeDeserializer& deserializer)>;

virtual size_t BeginList() = 0;
virtual void EndList() = 0;
using MapEntryConsumer = std::function<void(const Aws::String& key, ShapeDeserializer& deserializer)>;

virtual size_t BeginMap() = 0;
virtual void EndMap() = 0;
virtual void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) = 0;
virtual void ReadList(const Schema& schema, const ListElementConsumer& consumer) = 0;
virtual void ReadMap(const Schema& schema, const MapEntryConsumer& consumer) = 0;

virtual Aws::Crt::Optional<bool> ReadBoolean(const Schema& schema) = 0;
virtual Aws::Crt::Optional<int> ReadInteger(const Schema& schema) = 0;
virtual Aws::Crt::Optional<int64_t> ReadLong(const Schema& schema) = 0;
virtual Aws::Crt::Optional<float> ReadFloat(const Schema& schema) = 0;
virtual Aws::Crt::Optional<double> ReadDouble(const Schema& schema) = 0;
virtual Aws::Crt::Optional<Aws::String> ReadString(const Schema& schema) = 0;
virtual Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp(const Schema& schema) = 0;
virtual Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob(const Schema& schema) = 0;
virtual Aws::Crt::Optional<int> ReadEnum(const Schema& schema) = 0;

virtual Aws::Crt::Optional<Aws::String> ReadKey() = 0;
virtual bool IsBreak() = 0;
virtual bool IsNull() = 0;
virtual void ReadNull() = 0;
virtual void SkipValue() = 0;
};

} // namespace schema
} // namespace smithy
}
}
Loading
Loading