diff --git a/generator.py b/generator.py index 0041b70c8..29ff680ff 100755 --- a/generator.py +++ b/generator.py @@ -2,13 +2,14 @@ import argparse import time -from os.path import abspath, dirname, exists, join, realpath +from os.path import abspath, exists from jinja2 import TemplateNotFound from binary.util import binary_output_directories, test_output_directories from binary_generator import get_binary_templates, save_binary_files, save_test_files from util import * +from go import GoGenerator start = time.time() @@ -119,7 +120,21 @@ "should be performed. These checks are done by default.", ) +parser.add_argument( + "-s", + "--service", + action="append", + help="Specify the service to include. Can be used more than once to add more services.", +) + +parser.add_argument( + "--go-config", + help="(Go only) Specifies the Go generator configuration file. Implies '-l go'", +) + args = parser.parse_args() +if args.go_config: + args.lang = "go" lang = SupportedLanguages[args.lang.upper()] @@ -149,11 +164,6 @@ protocol_defs = sorted(protocol_defs, key=lambda proto_def: proto_def["id"]) -if not validate_services( - protocol_defs, schema_path, args.no_id_check, protocol_versions_as_numbers -): - exit(-1) - if custom_protocol_defs and not validate_custom_protocol_definitions( custom_protocol_defs, custom_codec_schema_path, protocol_versions_as_numbers ): @@ -161,8 +171,16 @@ print("Hazelcast Client Binary Protocol version", protocol_versions[-1]) +copy_verbatim_files(codec_output_dir, lang.value) + +if lang == SupportedLanguages.GO: + gen = GoGenerator(protocol_defs, custom_protocol_defs, codec_output_dir, args) + gen.generate() + exit(0) + env = create_environment(lang, args.namespace) + if lang != SupportedLanguages.MD: codec_template = env.get_template("codec-template.%s.j2" % lang.value) generate_codecs(protocol_defs, custom_protocol_defs, codec_template, codec_output_dir, lang, env) diff --git a/go/__init__.py b/go/__init__.py index e69de29bb..d3f0ccae9 100644 --- a/go/__init__.py +++ b/go/__init__.py @@ -0,0 +1 @@ +from .generator import GoGenerator diff --git a/go/codec-template.j2 b/go/codec-template.j2 new file mode 100644 index 000000000..c522f11bf --- /dev/null +++ b/go/codec-template.j2 @@ -0,0 +1,244 @@ +{% macro encode_var_sized(param, loop_last) -%} + {% if is_var_sized_list(param.type) or is_var_sized_list_contains_nullable(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}ListMultiFrame{% if is_var_sized_list_contains_nullable(param.type)%}ContainsNullable{% endif %}For{{ item_type(lang_name, param.type) }}(clientMessage, {{ escape_keyword(param.name) }}) + {%- elif is_var_sized_entry_list(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}EntryListFor{{ key_type(lang_name, param.type) }}And{{ value_type(lang_name, param.type) }}(clientMessage, {{ param_name(param.name) }}) + {%- elif is_var_sized_map(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}Map(clientMessage, {{ param_name(param.name) }}, Encode{{ key_type(lang_name, param.type) }}, Encode{{ value_type(lang_name, param.type) }}{% if loop_last %}, True{% endif %}) + {%- else -%} + {%- if param.nullable -%} + {%- if param.type|lower == "string" -%} + EncodeNullableForString(clientMessage, {{ param_name(param.name) }}) + {%- else -%} + EncodeNullable(clientMessage, {{ param_name(param.name) }}, Encode{{ lang_name(param.type) }}{% if loop_last %}, True{% endif %}) + {%- endif -%} + {%- else -%} + Encode{{ lang_name(param.type) }}(clientMessage, {{ param_name(param.name) }}{% if loop_last %}, True{% endif %}) + {%- endif %} + {% endif %} +{%- endmacro %} +{% macro decode_var_sized(param) -%} + {%- if is_var_sized_list(param.type) or is_var_sized_list_contains_nullable(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}ListMultiFrameFor{{ item_type(lang_name, param.type) }}{% if is_var_sized_list_contains_nullable(param.type) %}ContainsNullable{% endif %}(frameIterator) + {%- elif is_var_sized_entry_list(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}EntryListFor{{ key_type(lang_name, param.type) }}And{{ value_type(lang_name, param.type) }}(frameIterator) + {%- elif is_var_sized_map(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}Map(v, Decode{{ key_type(lang_name, param.type) }}, Decode{{ value_type(lang_name, param.type) }}) + {%- else -%} + {%- if param.nullable -%} + DecodeNullableFor{{ lang_name(param.type) }}(frameIterator) + {%- else -%} + Decode{{ lang_name(param.type) }}(frameIterator) + {%- endif -%} + {%- endif -%} +{%- endmacro %} +{% macro insert_import_statements(stmts) %} +import ( + {% for stmt in stmts %} + {{ stmt }} + {% endfor %} +) +{% endmacro %} +{% set request_fix_sized_params = fixed_params(method.request.params) %} +{% set request_var_sized_params = var_size_params(method.request.params) %} +{% set response_fix_sized_params = fixed_params(method.response.params) %} +{% set response_var_sized_params = var_size_params(method.response.params) %} +{% set response_new_params = new_params(method.since, method.response.params) %} +{% set event_fix_sized_params = [] %} +{% set event_var_sized_params = [] %} +{% if method.events|length != 0 %} + {% for event in method.events %} + {% do event_fix_sized_params.extend(fixed_params(event.params)) %} + {% do event_var_sized_params.extend(var_size_params(event.params)) %} + {% endfor %} +{% endif %} +/* +* Copyright (c) 2008-{{ copyright_year }}, Hazelcast, Inc. All Rights Reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License") +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package codec + +{{ insert_import_statements( + get_import_path_holders( + request_fix_sized_params, + request_var_sized_params, + response_fix_sized_params, + response_var_sized_params, + event_fix_sized_params, + event_var_sized_params + ) +)}} + +const( + {{ service_name|capital }}{{ method.name|capital }}CodecRequestMessageType = int32({{ '0x%06X'|format(method.request.id) }}) + {{ service_name|capital }}{{ method.name|capital }}CodecResponseMessageType = int32({{ '0x%06X'|format(method.response.id) }}) + +{% for event in method.events%} + {{ service_name|capital }}{{ method.name|capital }}CodecEvent{{ event.name|capital }}MessageType = int32({{ '0x%06X'|format(event.id) }}) + +{% endfor %} +{#FIXED SIZED PARAMETER OFFSET CONSTANTS#} +{% for param in request_fix_sized_params %} + {{ service_name|capital }}{{ method.name|capital }}CodecRequest{{ param.name|capital }}Offset = {% if loop.first %}proto.PartitionIDOffset + proto.IntSizeInBytes{% else %}{{ service_name|capital }}{{ method.name|capital }}CodecRequest{{ loop.previtem.name|capital }}Offset + proto.{{ loop.previtem.type|capitalize }}SizeInBytes{% endif %} + + {% if loop.last %} + {{ service_name|capital }}{{ method.name|capital }}CodecRequestInitialFrameSize = {{ service_name|capital }}{{ method.name|capital }}CodecRequest{{ param.name|capital }}Offset + proto.{{ param.type|capitalize }}SizeInBytes + + {% endif %} +{% else %} + {{ service_name|capital }}{{ method.name|capital }}CodecRequestInitialFrameSize = proto.PartitionIDOffset + proto.IntSizeInBytes + +{% endfor %} +{% for param in response_fix_sized_params %} + {{ service_name|capital }}{{ method.name|capital }}Response{{ param.name|capital }}Offset = {% if loop.first %}proto.ResponseBackupAcksOffset + proto.ByteSizeInBytes{% else %}{{ service_name|capital }}{{ method.name|capital }}Response{{ loop.previtem.name|capital }}Offset + proto.{{ loop.previtem.type|capitalize }}SizeInBytes{% endif %} + +{% endfor %} +{% for event in method.events %} + {% for param in fixed_params(event.params) %} + {{ service_name|capital }}{{ method.name|capital }}Event{{ event.name|capital }}{{param.name|capital}}Offset = {% if loop.first %}proto.PartitionIDOffset + proto.IntSizeInBytes{% else %}{{ service_name|capital }}{{ method.name|capital }}Event{{ event.name|capital }}{{ loop.previtem.name|capital }}Offset + proto.{{loop.previtem.type|capitalize}}SizeInBytes{% endif %} + + {% endfor %} +{% endfor %} +) +{#RESPONSE PARAMETERS#} +{% set noResponseValue = method.response.params|length == 0 %} +{% set singleResponseValue = method.response.params|length == 1 and response_new_params|length == 0 %} + +{% for line in method.doc.splitlines() %} +// {{ line }} +{% endfor %} + +func Encode{{ service_name|capital }}{{ method.name|capital }}Request({% for param in method.request.params %}{{ escape_keyword(param.name) }} {{ lang_types_encode(param.type) }}{% if not loop.last %}, {% endif %}{% endfor %}) *proto.ClientMessage { + clientMessage := proto.NewClientMessageForEncode() + clientMessage.SetRetryable({{ method.request.retryable|lower }}) + + initialFrame := proto.NewFrameWith(make([]byte, {{ service_name|capital }}{{ method.name|capital }}CodecRequestInitialFrameSize), proto.UnfragmentedMessage) +{% for param in request_fix_sized_params %} + Encode{{ param.type|capital }}(initialFrame.Content, {{ service_name|capital }}{{ method.name|capital }}CodecRequest{{ param.name|capital }}Offset, {{ escape_keyword(param.name) }}) +{% endfor %} + clientMessage.AddFrame(initialFrame) + clientMessage.SetMessageType({{ service_name|capital }}{{ method.name|capital }}CodecRequestMessageType) + clientMessage.SetPartitionId(-1) + +{% for param in request_var_sized_params %} + {{ encode_var_sized(param) }} +{% endfor %} + + return clientMessage +} + +{#RESPONSE DECODE#} +{% if noResponseValue %} +{% elif singleResponseValue %} + {% set param = method.response.params|last %} +func Decode{{ service_name|capital }}{{ method.name|capital }}Response(clientMessage *proto.ClientMessage) {{ lang_types_decode(param.type) }} { + frameIterator := clientMessage.FrameIterator() +{% if response_fix_sized_params|length != 0 %} + initialFrame := frameIterator.Next() +{% else %} + // empty initial frame + frameIterator.Next() +{% endif %} + +{% for param in response_fix_sized_params %} + return Decode{{ param.type|capital }}(initialFrame.Content, {{ service_name|capital }}{{ method.name|capital }}Response{{param.name|capital}}Offset) +{% endfor %} +{% for param in response_var_sized_params %} + return {{ decode_var_sized(param) }} +{% endfor %} +} +{% else %} +func Decode{{ service_name|capital }}{{ method.name|capital }}Response(clientMessage *proto.ClientMessage) ({% for param in method.response.params %}{{ escape_keyword(param.name)}} {{ lang_types_encode(param.type) }}{{ ", " if not loop.last }}{% endfor %}{% for new_param in response_new_params %} is{{ new_param.name|capital }}Exists bool{% endfor %}) { +{% if response_fix_sized_params|length != 0 %} + frameIterator := clientMessage.FrameIterator() + initialFrame := frameIterator.Next() +{% else %} + frameIterator := clientMessage.FrameIterator() + frameIterator.Next() +{% endif %} + +{% for param in response_fix_sized_params %} + {% if param in response_new_params %} + if (initialFrame.content.length >= RESPONSE_{{ to_upper_snake_case(param.name) }}_OFFSET + BitsUtil.{{ param.type.upper() }}_SIZE_IN_BYTES) { + response.{{ escape_keyword(param.name) }} = Decode{{ param.type|capital }}(initialFrame.Content, RESPONSE_{{to_upper_snake_case(param.name)}}_OFFSET); + response.is{{ param.name|capital }}Exists = true; + } else { + response.is{{ param.name|capital }}Exists = false; + } + {% else %} + {{ escape_keyword(param.name) }} = Decode{{ param.type|capital }}(initialFrame.Content, {{ service_name|capital }}{{ method.name|capital }}Response{{ param.name|capital }}Offset) + {% endif %} +{% endfor %} +{% for param in response_var_sized_params %} + {% if param in response_new_params %} + if (clientMessage.hasNextFrame()) { + response.{{ escape_keyword(param.name) }} = {{ decode_var_sized(param) }}; + response.is{{ param.name|capital }}Exists = true; + } else { + response.is{{ param.name|capital }}Exists = false; + } + {% else %} + {{ escape_keyword(param.name) }} = {{ decode_var_sized(param) }} + {% endif %} +{% endfor %} + + return {% for param in method.response.params %}{{ escape_keyword(param.name)}}{{ ", " if not loop.last }}{% endfor %} {% for new_param in response_new_params %} is{{ new_param.name|capital }}Exists bool{% endfor %} +} +{% endif %} +{# EVENTS#} +{% if method.events|length != 0 %} + +func Handle{{ service_name|capital }}{{ method.name|capital }}(clientMessage *proto.ClientMessage, {% for event in method.events%}handle{{ event.name|capital }}Event func({% for param in event.params %}{{param.name}} {{ lang_types_encode(param.type) }}{% if not loop.last %}, {% endif %}{% endfor %}){% if not loop.last %}, {% endif %}{% endfor %}){ + messageType := clientMessage.Type() + frameIterator := clientMessage.FrameIterator() + {% for event in method.events%} + if messageType == {{ service_name|capital }}{{ method.name|capital }}CodecEvent{{ event.name|capital }}MessageType { + {% set new_event_params = new_params(event.since, event.params) %} + {% if fixed_params(event.params)|length != 0 %} + initialFrame := frameIterator.Next() + {% else %} + //empty initial frame + frameIterator.Next() + {% endif %} + {% for param in fixed_params(event.params) %} + {% if param in new_event_params %} + let is{{ param.name|capital }}Exists = false + let {{ escape_keyword(param.name) }} = {% if param.type == 'boolean' %}false{% elif param.type == 'UUID' %}null{% else %}0{% endif %} + if (initialFrame.content.length >= EVENT_{{ to_upper_snake_case(event.name)}}_{{ to_upper_snake_case(param.name) }}_OFFSET + BitsUtil.{{ param.type.upper() }}_SIZE_IN_BYTES) { + {{ escape_keyword(param.name) }} = Decode{{ param.type|capital }}(initialFrame.content, EVENT_{{ to_upper_snake_case(event.name)}}_{{ to_upper_snake_case(param.name) }}_OFFSET) + is{{ param.name|capital }}Exists = true; + } + {% else %} + {{ escape_keyword(param.name) }} := Decode{{ param.type|capital }}(initialFrame.Content, {{ service_name|capital }}{{ method.name|capital }}Event{{ event.name|capital }}{{param.name|capital}}Offset) + {% endif %} + {% endfor %} + {% for param in var_size_params(event.params) %} + {% if param in new_event_params %} + let is{{ param.name|capital }}Exists = false + let {{ escape_keyword(param.name) }} = null + if (iterator.hasNextFrame()) { + {{ escape_keyword(param.name) }} = {{ decode_var_sized(param) }} + is{{ param.name|capital }}Exists = true; + } + {% else %} + {{ escape_keyword(param.name) }} := {{ decode_var_sized(param) }} + {% endif %} + {% endfor %} + handle{{ event.name|capital }}Event({% for param in event.params %}{% if param in new_event_params %}is{{ param.name|capital }}Exists, {% endif %}{{param.name}}{% if not loop.last %}, {% endif %}{% endfor %}) + return + } + {% endfor %} +} +{% endif %} diff --git a/go/config.json b/go/config.json new file mode 100644 index 000000000..8b3d72426 --- /dev/null +++ b/go/config.json @@ -0,0 +1,13 @@ +{ + "include_methods": ["Map.*"], + "include_types": ["Address"], + "mapping": { + "Address": "pubcluster.Address", + "longArray": "[]int64", + "List_String": "[]string", + "List_UUID": "[]types.UUID", + "List_Long": "[]int64", + "EntryList_String_EntryList_Integer_Long": "map[string]map[int32]int64", + "EntryList_Integer_UUID": "map[int32]types.UUID" + } +} \ No newline at end of file diff --git a/go/config.py b/go/config.py new file mode 100644 index 000000000..e1e1e920b --- /dev/null +++ b/go/config.py @@ -0,0 +1,25 @@ + +DEFAULT_IMPORT_PATH = "github.com/hazelcast/hazelcast-go-client" + + +class Config: + + def __init__(self, base_import_path="", include_methods=None, include_types=None, mapping=None, override_imports=None, types=None): + self.base_import_path = base_import_path or DEFAULT_IMPORT_PATH + self.include_methods = include_methods or [] + self.include_types = include_types or [] + self.mapping = mapping or {} + self.override_imports = override_imports or {} + self.types = types or {} + + @classmethod + def from_json(cls, d: dict): + keys = [ + "base_import_path", + "include_methods", + "include_types", + "mapping", + "override_imports", + "types" + ] + return cls(**{k: d.get(k) for k in keys}) \ No newline at end of file diff --git a/go/custom-codec-template.j2 b/go/custom-codec-template.j2 new file mode 100644 index 000000000..5f9683b8c --- /dev/null +++ b/go/custom-codec-template.j2 @@ -0,0 +1,183 @@ +{% macro encode_var_sized(param) -%} + {% if is_var_sized_list(param.type) or is_var_sized_list_contains_nullable(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}ListMultiFrame{% if is_var_sized_list_contains_nullable(param.type)%}ContainsNullable{% endif %}For{{ item_type(lang_name, param.type) }}(clientMessage, {{ param_name(codec.name)}}.{{ param.name|capital }}) + {%- elif is_var_sized_entry_list(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}EntryListFor{{ value_type(lang_name, param.type) }}And{{ key_type(lang_name, param.type) }}(clientMessage, {{ param_name(codec.name)}}.{{ param.name|capital }}) + {%- elif is_var_sized_map(param.type) -%} + Encode{% if param.nullable %}Nullable{% endif %}MapFor{{ key_type(lang_name, param.type) }}And{{ value_type(lang_name, param.type) }}(clientMessage, {{ param_name(codec.name)}}.{{ param.name|capital }}) + {%- else -%} + {%- if param.nullable -%} + EncodeNullableFor{{ lang_name(param.type) }}(clientMessage, {% if param.type != "String" %}&{% endif %}{{ param_name(codec.name)}}.{{ param.name|capital }}) + {%- else -%} + Encode{{ lang_name(param.type) }}(clientMessage, {{ param_name(codec.name)}}.{{ param.name|capital }}) + {%- endif %} + {% endif %} +{%- endmacro %} +{% macro decode_var_sized(param) -%} + {%- if is_var_sized_list(param.type) or is_var_sized_list_contains_nullable(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}ListMultiFrame{% if is_var_sized_list_contains_nullable(param.type) %}ContainsNullable{% endif %}For{{ item_type(lang_name, param.type) }}(frameIterator) + {%- elif is_var_sized_entry_list(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}EntryList(frameIterator, Decode{{ key_type(lang_name, param.type) }}, DecodeNullable{{ value_type(lang_name, param.type) }}) + {%- elif is_var_sized_map(param.type) -%} + Decode{% if param.nullable %}Nullable{% endif %}MapFor{{ key_type(lang_name, param.type) }}And{{ value_type(lang_name, param.type) }}(frameIterator) + {%- else -%} + {%- if param.nullable -%} + DecodeNullableFor{{ lang_name(param.type) }}(frameIterator) + {%- else -%} + Decode{{ lang_name(param.type) }}(frameIterator) + {%- endif -%} + {%- endif -%} +{%- endmacro %} +{% macro insert_import_statements(stmts) %} +import ( + {% for stmt in stmts %} + {{ stmt }} + {% endfor %} +) +{% endmacro %} +{% set fix_sized_params = fixed_params(codec.params) %} +{% set var_sized_params = var_size_params(codec.params) %} +{% set new_codec_params = new_params(codec.since, codec.params) %} +{% set fix_sized_new_params = new_params(codec.since, fix_sized_params) %} +{% set should_add_begin_frame = (fix_sized_params|length > fix_sized_new_params|length) or fix_sized_params|length == 0 %} +/* +* Copyright (c) 2008-{{ copyright_year }}, Hazelcast, Inc. All Rights Reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License") +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package codec + +{{ insert_import_statements( + get_import_path_holders( + codec.name, + fix_sized_params, + var_sized_params + ) +)}} + +{% if fix_sized_params|length > 0 %} +const( +{% for param in fix_sized_params %} + {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset = {% if loop.first %}0{% else %}{{ codec.name|capital }}Codec{{ loop.previtem.name|capital }}FieldOffset + proto.{{ loop.previtem.type|capital }}SizeInBytes{% endif %} + + {% if loop.last %} + {{ codec.name|capital }}Codec{{ param.name|capital }}InitialFrameSize = {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset + proto.{{ param.type|capital }}SizeInBytes + {% endif %} +{% endfor %} +) +{% endif %} + +{% if codec.name == "Address" %} +{% else %} +func Encode{{ codec.name|capital }}(clientMessage *proto.ClientMessage, {{ param_name(codec.name) }} {{ lang_types_encode(codec.name) }}){ +{% if should_add_begin_frame %} + clientMessage.AddFrame(proto.BeginFrame.Copy()) +{% endif %} +{% for param in fix_sized_params %} + {% if loop.last %} + initialFrame := proto.NewFrame(make([]byte,{{ codec.name|capital }}Codec{{ param.name|capital }}InitialFrameSize)) + {% endif %} +{% endfor %} +{% for param in fix_sized_params %} + {% if escape_keyword(param.name).startswith('_') == true %} + {% set keyword = escape_keyword(param.name)[1:] %} + {% else %} + {% set keyword = escape_keyword(param.name) %} + {% endif %} + {% if param.type == "int" %} + {% set cast = "int32(" %} + {% elif param.type == "long" %} + {% set cast = "int64(" %} + {% else %} + {% set cast = "" %} + {% endif %} + Encode{{ param.type|capital }}(initialFrame.Content, {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset, {% if cast %}{{ cast }}{% endif %}{{ param_name(codec.name)}}.{{ rename_field(codec.name, param) }}{% if cast %}){% endif %}) + {% if loop.last %} + {% if not should_add_begin_frame %} + initialFrame.flags |= BEGIN_DATA_STRUCTURE_FLAG + {% endif %} + clientMessage.AddFrame(initialFrame) + {% endif %} +{% endfor %} +{% for param in var_sized_params %} + {% if loop.first %} + + {% endif %} + {% if lang_types_encode(param.type) == '!skip' %} + {% continue %} + {% endif %} + {{ encode_var_sized(param) }} +{% endfor %} + + clientMessage.AddFrame(proto.EndFrame.Copy()) +} +{% endif %} + +func Decode{{ codec.name|capital }}(frameIterator *proto.ForwardFrameIterator) {{ lang_types_encode(codec.name) }} { + // begin frame +{% if should_add_begin_frame %} + frameIterator.Next() +{% endif %} +{% for param in fix_sized_params %} + {% if loop.first %} + initialFrame := frameIterator.Next() + {% endif %} + {% if param in fix_sized_new_params %} + var {{ escape_keyword(param.name) }} {{ lang_types_encode(param.type) }} + if len(initialFrame.Content) >= {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset + proto.{{ param.type|capitalize }}SizeInBytes { + {{ escape_keyword(param.name) }} = Decode{{ param.type|capital }}(initialFrame.Content, {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset) + } + {% else %} + {{ escape_keyword(param.name) }} := Decode{{ param.type|capital }}(initialFrame.Content, {{ codec.name|capital }}Codec{{ param.name|capital }}FieldOffset) + {% endif %} +{% endfor %} +{% for param in var_sized_params %} + {% if loop.first %} + + {% endif %} + {% if lang_types_decode(param.type) == '!skip' %} + {% continue %} + {% endif %} + {% if param in new_codec_params %} + var {{ escape_keyword(param.name) }} interface{} + if (!frameIterator.PeekNext().IsEndFrame()) { + {{ escape_keyword(param.name) }} = {{ decode_var_sized(param) }} + } + {% else %} + {{ escape_keyword(param.name) }} := {{ decode_var_sized(param) }} + {% endif %} +{% endfor %} + FastForwardToEndFrame(frameIterator) +{% set ctor_params = [] %} +{% for param in codec.params %} + {% if lang_types_decode(param.type) != '!skip' %} + {% do ctor_params.append(param) %} + {% endif %} +{% endfor %} + {% set encoded_codec_name = lang_types_encode(codec.name) %} + {% if encoded_codec_name.startswith("*") %} + {% set encoded_codec_name = encoded_codec_name[1:] %} + {% endif %} + {% if codec.name == "Address"%} + return pubcluster.NewAddress(host, port) + {% else %} + return {% if "*" in lang_types_encode(codec.name) %}&{% endif %}{{ encoded_codec_name.split('.')[0] }}.{{ encoded_codec_name.split('.')[1] }}{% raw %}{{%endraw %} + + {% for param in ctor_params %} + {# {% if param in new_codec_params %}is{{ param.name|capital }}Exists, {% endif %} #} + {{ rename_field(codec.name, param) }}: {{ augment_enum(codec.name, param) }}, + {% endfor %} + } + {% endif %} +} diff --git a/go/generator.py b/go/generator.py new file mode 100644 index 000000000..ecf1891ea --- /dev/null +++ b/go/generator.py @@ -0,0 +1,370 @@ +import hashlib +import json +import os +import fnmatch +import re +from datetime import datetime + +from jinja2 import Environment, PackageLoader + +from cpp import get_size, is_trivial +from py import py_param_name +from util import capital, to_upper_snake_case, is_var_sized_list_contains_nullable, \ + fixed_params, var_size_params, new_params, filter_new_params, is_var_sized_list, is_var_sized_entry_list, \ + is_var_sized_map, item_type, key_type, value_type, param_name, java_name +from .config import Config + + +RE_PKG_TYPE = re.compile(r"([a-z]+)\.(\w+)") + + +class GoGenerator: + + codec_filename_gen = lambda *names: f'{"_".join(map(py_param_name, names))}_codec.go' + + def __init__(self, services, custom_protocol, output_dir, args): + self.services = services + self.custom_types = custom_protocol[0]["customTypes"] + self.output_dir = output_dir + self.args = args + config_path = args.go_config + self.extensibility = True + if not config_path: + config_path = os.path.join(current_path(), "config.json") + self.extensibility = False + self.config = self.load_config(config_path) + self.mapping = DEFAULT_MAPPING.copy() + if self.config.mapping: + self.mapping.update(self.config.mapping) + types = DEFAULT_TYPES.copy() + merge_dict(types, self.config.types) + self.types_encode = self.make_types_encode(DEFAULT_TYPES) + self.types_decode = self.make_types_decode(DEFAULT_TYPES) + self.env = self.make_environment() + + def generate(self): + for service in self.services: + self.generate_service(service) + self.generate_type_codecs(self.custom_types) + self.generate_types(self.custom_types) + + def generate_service(self, service): + template = self.env.get_template("codec-template.j2") + methods = service.get("methods") + if methods is None: + raise NotImplementedError(f"Methods not found for service {service}") + for method in methods: + self.generate_method(service, method, template) + + def generate_method(self, service, method, template): + name = f"{service['name']}.{method['name']}" + if not self.can_generate_method(name): + return + opts = {"base_import_path": self.config.base_import_path} + method["request"]["id"] = self.make_id(service["id"], method["id"], 0) + method["response"]["id"] = self.make_id(service["id"], method["id"], 1) + for i, event in enumerate(method.get("events", [])): + event["id"] = self.make_id(service["id"], method["id"], i + 2) + codec_file_name = GoGenerator.codec_filename_gen(service["name"], method["name"]) + try: + content = template.render(service_name=service["name"], method=method, **opts) + path = os.path.join(self.output_dir, codec_file_name) + self.save(path, content) + except NotImplementedError as e: + print(f"{codec_file_name} contains missing type mapping so ignoring it. Error: {e}") + + def generate_type_codecs(self, codecs): + template = self.env.get_template("custom-codec-template.j2") + for codec in codecs: + self.generate_type_codec(codec, template) + + def generate_type_codec(self, codec, template): + if not self.can_generate_type(codec["name"]): + return + opts = {"base_import_path": self.config.base_import_path} + fname = GoGenerator.codec_filename_gen(codec["name"]) + content = template.render(codec=codec, **opts) + path = os.path.join(self.output_dir, fname) + self.save(path, content) + + def generate_types(self, codecs): + template = self.env.get_template("type-template.j2") + for codec in codecs: + self.generate_type(codec, template) + + def generate_type(self, codec, template): + codec_name = codec["name"] + if not self.can_generate_type(codec_name): + return + to_t = self.mapping.get(codec_name) + if not to_t: + print(f"Mapping for type {codec_name} not found") + return + pkg_t = RE_PKG_TYPE.search(to_t) + if not pkg_t: + print(f"Not a custom type: {to_t}") + return + pkg, t = pkg_t.group(1), pkg_t.group(2) + path = "" + if pkg.startswith("pub"): + pkg = pkg[3:] + path = pkg + elif pkg.startswith("i"): + pkg = pkg[1:] + path = f"internal/{pkg}" + path = os.path.join(self.output_dir, path) + os.makedirs(path, exist_ok=True) + fname = f'{py_param_name(codec_name)}.go' + content = template.render(codec=codec, package_name=pkg) + path = os.path.join(path, fname) + self.save(path, content) + + def can_generate_method(self, method_name: str) -> bool: + for filter in self.config.include_methods: + if fnmatch.fnmatch(method_name.lower(), filter.lower()): + return True + return False + + def can_generate_type(self, type_name: str) -> bool: + for filter in self.config.include_types: + if fnmatch.fnmatch(type_name.lower(), filter.lower()): + return True + return False + + @classmethod + def load_config(cls, path: str) -> Config: + with open(path) as f: + d = json.loads(f.read()) + return Config.from_json(d) + + @classmethod + def render(cls, service, template, method, **opts): + return template.render(service_name=service["name"], method=method, **opts) + + @classmethod + def save(cls, path: str, content: str, mode="w"): + m = hashlib.md5() + m.update(content.encode("utf-8")) + codec_hash = m.hexdigest() + with open(path, mode, newline=os.linesep) as path: + path.writelines(content.replace("!codec_hash!", codec_hash)) + + @classmethod + def make_id(cls, service_id, method_id, seq) -> int: + return int(f"0x{service_id:02x}{method_id:02x}{seq:02x}", 16) + + def make_environment(self): + env = Environment( + loader=PackageLoader("go", current_path()), + extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"], + ) + env.trim_blocks = True + env.lstrip_blocks = True + env.keep_trailing_newline = False + env.filters["capital"] = capital + env.globals["to_upper_snake_case"] = to_upper_snake_case + env.globals["fixed_params"] = fixed_params + env.globals["var_size_params"] = var_size_params + env.globals["new_params"] = new_params + env.globals["filter_new_params"] = filter_new_params + env.globals["is_var_sized_list"] = is_var_sized_list + env.globals["is_var_sized_list_contains_nullable"] = is_var_sized_list_contains_nullable + env.globals["is_var_sized_entry_list"] = is_var_sized_entry_list + env.globals["is_var_sized_map"] = is_var_sized_map + env.globals["item_type"] = item_type + env.globals["key_type"] = key_type + env.globals["value_type"] = value_type + env.globals["lang_types_encode"] = self.go_types_encode + env.globals["lang_types_decode"] = self.go_types_decode + env.globals["lang_name"] = java_name + env.globals["param_name"] = param_name + env.globals["escape_keyword"] = go_escape_keyword + env.globals["get_size"] = get_size + env.globals["is_trivial"] = is_trivial + env.globals["get_import_path_holders"] = self.go_get_import_statements + env.globals["augment_enum"] = go_augment_enum + env.globals["rename_field"] = go_rename_field + env.globals["copyright_year"] = datetime.now().year + return env + + def make_import_stmts_mapping(self, types: dict) -> dict: + bip = self.config.base_import_path + stmts = {} + for package, ts in types.get("public", {}).items(): + for t in ts: + stmts[t] = f"/{package}" + for package, ts in types.get("internal", {}).items(): + for t in ts: + stmts[t] = f"/internal/{package}" + for t, path in stmts.items(): + pp = path.split("/") + package = pp[-1] + alias = f"pub{package}" + if "internal" in pp: + alias = f"i{package}" + stmts[t] = f'{alias} "{bip}{path}"' + if self.extensibility: + stmts["Data"] = 'iserialization "github.com/hazelcast/hazelcast-go-client"' + return stmts + + def make_types_encode(self, types: dict): + res = {} + for package, ts in types.get("public", {}).items(): + for t in ts: + res[t] = f"pub{package}.{t}" + for package, ts in types.get("internal", {}).items(): + for t in ts: + res[t] = f"i{package}.{t}" + for from_type, to_type in self.mapping.items(): + res[from_type] = to_type + # if self.extensibility: + # res["Data"] = "hazelcast.Data" + return res + + make_types_decode = make_types_encode + + # def make_types_decode(self, types: dict): + # res = {} + # for package, ts in types.get("public", {}).items(): + # for t in ts: + # res[t] = f"pub{package}.{t}" + # for package, ts in types.get("internal", {}).items(): + # for t in ts: + # res[t] = f"i{package}.{t}" + # for from_type, to_type in self.config.mapping.items(): + # res[from_type] = to_type + # if self.extensibility: + # res["Data"] = "hazelcast.Data" + # return res + + def go_get_import_statements(self, *args): + import_map = self.make_import_stmts_mapping(DEFAULT_TYPES) + import_statements = set() + for arg in args: + params = [arg] if isinstance(arg, str) else arg + for param in params: + type = param["type"] if isinstance(param, dict) else param + import_stmt = import_map.get(type) + if import_stmt is None: + # this may be a mapped type + mt = self.mapping.get(type) + if mt: + p = mt.split(".", 1) + if len(p) == 2: + import_stmt = import_map.get(p[1]) + if import_stmt: + import_statements.add(import_stmt) + if not import_stmt: + continue + # TODO: adapt the proto import path according to GOCE. + if self.extensibility: + import_statements.add('proto "github.com/hazelcast/hazelcast-go-client"') + else: + import_statements.add('"github.com/hazelcast/hazelcast-go-client/internal/proto"') + # if args[0] == "SimpleEntryView": + # import_statements.remove('iserialization"github.com/hazelcast/hazelcast-go-client/internal/serialization"') + return sorted(import_statements) + + def go_types_encode(self, key): + try: + return self.types_encode[key] + except KeyError: + raise NotImplementedError(f"Missing type mapping for '{key}'") + + def go_types_decode(self, key): + try: + return self.types_decode[key] + except KeyError: + raise NotImplementedError(f"Missing type mapping for '{key}'") + + +def current_path(): + return os.path.split(os.path.abspath(__file__))[0] + + +go_reserved_keywords = {"break", "default", "func", "interface", "select", "case", "defer", "go", "map", "struct", + "chan", "else", "goto", "package", "switch", "const", "fallthrough", "if", "range", "type", + "continue", "for", "import", "return", "var"} + +DEFAULT_TYPES = { + "builtin": [ + "ByteArrayCodec", "CodecUtil", "DataCodec", + ], + "public": { + "cluster": ["Address", "EndpointQualifier", "MemberInfo", "MemberVersion"], + "sql": ["ListSqlColumnMetadata", "SqlColumnMetadata"], + "types": [ + "BitmapIndexOptions", "DistributedObjectInfo", "IndexConfig", + "SimpleEntryView", "UUID" + ], + }, + "internal": { + "hzerrors": ["StackTraceElement"], + "serialization": ["Data"], + "sql": ["Error", "Page", "QueryID"] + } +} + + +_go_enum = { + ("BitmapIndexOptions", "uniqueKeyTransformation"): "types.UniqueKeyTransformation", + ("EndpointQualifier", "type"): "cluster.EndpointQualifierType", + ("IndexConfig", "type"): "types.IndexType", +} + +_go_field = { + ("SimpleEntryView", "ttl"): "TTL", + ("MemberInfo", "uuid"): "UUID", +} + + +def go_escape_keyword(value): + if value not in go_reserved_keywords: + return value + return "_%s" % value + + +def go_augment_enum(codec, param): + cast_type = _go_enum.get((codec, param["name"])) + if cast_type: + return "%s(%s)" % (cast_type, go_escape_keyword(param["name"])) + return go_escape_keyword(param["name"]) + + +def go_rename_field(codec, param): + name = param["name"] + field_name = _go_field.get((codec, name)) + if field_name: + return field_name + return "%s%s" % (name[0].upper(), name[1:]) + + +def merge_dict(d1, d2: dict): + root = d1 + for k, v in d2.items(): + if k in d1: + if isinstance(d1[k], dict) and isinstance(v, dict): + merge_dict(d1[k], v) + continue + root[k] = v + + +DEFAULT_MAPPING = { + "Address": "", + "boolean": "bool", + "byte": "byte", + "EntryList_Data_Data": "[]proto.Pair", + "EntryList_Integer_Integer": "[]proto.Pair", + "int": "int32", + "ListCN_Data": "[]iserialization.Data", + "List_Data": "[]iserialization.Data", + "List_IndexConfig": "[]pubtypes.IndexConfig", + "List_ListCN_Data": "[]iserialization.Data", + "List_QueryCacheConfigHolder": "[]QueryCacheConfigHolder", + "List_SqlColumnMetadata": "[]isql.ColumnMetadata", + "long": "int64", + "SqlError": "isql.Error", + "SqlPage": "isql.Page", + "SqlQueryId": "isql.QueryID", + "String": "string", +} diff --git a/go/codec-template.go.j2 b/go/tests/__init__.py similarity index 100% rename from go/codec-template.go.j2 rename to go/tests/__init__.py diff --git a/go/tests/test_go_generator.py b/go/tests/test_go_generator.py new file mode 100644 index 000000000..31f6c77b2 --- /dev/null +++ b/go/tests/test_go_generator.py @@ -0,0 +1,113 @@ +import unittest +from collections import namedtuple + +from go.generator import merge_dict + + +class GoGeneratorTestCase(unittest.TestCase): + + def test_merge_dict(self): + case = namedtuple("case", ["msg", "a", "b", "target"]) + test_cases = [ + # case( + # msg="a and b are empty", + # a={}, + # b={}, + # target={}, + # ), + # case( + # msg="only a is empty", + # a={}, + # b={"k1": "v1"}, + # target={"k1": "v1"}, + # ), + # case( + # msg="only b is empty", + # a={"k1": "v1"}, + # b={}, + # target={"k1": "v1"}, + # ), + # case( + # msg="both a and b are simple dicts", + # a={"ka1": "va1", "k1": "v1"}, + # b={"kb1": "vb1", "k1": "v2"}, + # target={"ka1": "va1", "kb1": "vb1", "k1": "v2"} + # ), + # case( + # msg="a is nested dict, b is simple dict", + # a={"ka1": "va1", "k1": {"ka21", "va21"}}, + # b={"kb1": "vb1", "k1": "v2"}, + # target={"ka1": "va1", "kb1": "vb1", "k1": "v2"} + # ), + # case( + # msg="a is simple dict, b is nested dict", + # a={"ka1": "va1", "k1": "v1"}, + # b={"kb1": "vb1", "k1": {"kb21", "vb21"}}, + # target={"ka1": "va1", "kb1": "vb1", "k1": {"kb21", "vb21"}} + # ), + # case( + # msg="both a and be are non-overlapping nested dicts", + # a={"ka1": "va1", "ka2": {"ka21", "va21"}, "k1": "v1"}, + # b={"kb1": "vb1", "kb2": {"kb21", "vb21"}, "k1": "v1"}, + # target={"ka1": "va1", "ka2": {"ka21", "va21"}, "kb2": {"kb21", "vb21"}, "kb1": "vb1", "k1": "v1"} + # ), + case( + msg="both a and be are overlapping nested dicts", + a={"k2": {"k21": "va21", "ka21": "va21"}}, + b={"k2": {"k21": "vb21", "kb21": "vb21"}}, + target={"k2": {"k21": "vb21", "ka21": "va21", "kb21": "vb21"}} + ), + case( + msg="", + a={ + "builtin": [ + "ByteArrayCodec", "CodecUtil", "DataCodec", + ], + "public": { + "cluster": ["Address", "EndpointQualifier", "MemberInfo", "MemberVersion"], + "sql": ["ListSqlColumnMetadata", "SqlColumnMetadata"], + "types": [ + "BitmapIndexOptions", "DistributedObjectInfo", "IndexConfig", + "SimpleEntryView", "UUID" + ], + }, + "internal": { + "hzerrors": ["StackTraceElement"], + "serialization": ["Data"], + "sql": ["Error", "Page", "QueryID"] + } + }, + b={ + "public": { + "control": ["EventJournalConfig"] + } + }, + target={ + "builtin": [ + "ByteArrayCodec", "CodecUtil", "DataCodec", + ], + "public": { + "cluster": ["Address", "EndpointQualifier", "MemberInfo", "MemberVersion"], + "control": ["EventJournalConfig"], + "sql": ["ListSqlColumnMetadata", "SqlColumnMetadata"], + "types": [ + "BitmapIndexOptions", "DistributedObjectInfo", "IndexConfig", + "SimpleEntryView", "UUID" + ], + }, + "internal": { + "hzerrors": ["StackTraceElement"], + "serialization": ["Data"], + "sql": ["Error", "Page", "QueryID"] + } + } + ) + ] + for case in test_cases: + with self.subTest(msg=case.msg): + merge_dict(case.a, case.b) + self.assertEqual(case.a, case.target) + + +if __name__ == '__main__': + unittest.main() diff --git a/go/tests/testdata/hzpo-config.json b/go/tests/testdata/hzpo-config.json new file mode 100644 index 000000000..6fbfd33ef --- /dev/null +++ b/go/tests/testdata/hzpo-config.json @@ -0,0 +1,24 @@ +{ + "base_import_path": "github.com/yuce/listindexes", + "include_methods": ["MC.getMapConfig"], + "include_types": ["Address", "EventJournalConfig", "IndexConfig"], + "types": { + "public": { + "control": ["EventJournalConfig", "Address", "IndexConfig"] + } + }, + "mapping": { + "Address": "pubcontrol.Adddress", + "EventJournalConfig": "pubcontrol.EventJournalConfig", + "HotRestartConfig": "pubcontrol.HotRestartConfig", + "EvictionConfigHolder": "pubcontrol.EvictionConfigHolder", + "List_AttributeConfig": "[]pubcontrol.AttributeConfig", + "List_ListenerConfigHolder": "[]pubcontrol.ListenerConfigHolder", + "MapStoreConfigHolder": "pubcontrol.MapStoreConfigHolder", + "MerkleTreeConfig": "pubcontrol.MerkleTreeConfig", + "NearCacheConfigHolder": "pubcontrol.NearCacheConfigHolder", + "WanReplicationRef": "pubcontrol.WanReplicationRef", + "List_String": "[]string", + "IndexConfig": "pubtypes.IndexConfig" + } +} \ No newline at end of file diff --git a/go/type-template.j2 b/go/type-template.j2 new file mode 100644 index 000000000..c1c9595d3 --- /dev/null +++ b/go/type-template.j2 @@ -0,0 +1,23 @@ +/* +* Copyright (c) 2008-{{ copyright_year }}, Hazelcast, Inc. All Rights Reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License") +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package {{ package_name }} + +type {{ codec.name }} struct { +{% for param in codec.params %} + {{ escape_keyword(param.name)|capital }} {{ lang_types_encode(param.type) }} +{% endfor -%} +} \ No newline at end of file diff --git a/go/verbatim/builtin.go b/go/verbatim/builtin.go new file mode 100644 index 000000000..95c627bdb --- /dev/null +++ b/go/verbatim/builtin.go @@ -0,0 +1,820 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package codec + +import ( + "encoding/binary" + "fmt" + "net" + "strconv" + "strings" + + pubcluster "github.com/hazelcast/hazelcast-go-client/cluster" + ihzerrors "github.com/hazelcast/hazelcast-go-client/internal/hzerrors" + "github.com/hazelcast/hazelcast-go-client/internal/proto" + iserialization "github.com/hazelcast/hazelcast-go-client/internal/serialization" + "github.com/hazelcast/hazelcast-go-client/types" +) + +// Encoder for ClientMessage and value +type Encoder func(message *proto.ClientMessage, value interface{}) + +// Decoder create *iserialization.Data +type Decoder func(frameIterator *proto.ForwardFrameIterator) *iserialization.Data + +// CodecUtil +type codecUtil struct{} + +var CodecUtil codecUtil + +func (codecUtil) FastForwardToEndFrame(frameIterator *proto.ForwardFrameIterator) { + numberOfExpectedEndFrames := 1 + var frame *proto.Frame + for numberOfExpectedEndFrames != 0 { + frame = frameIterator.Next() + if frame.IsEndFrame() { + numberOfExpectedEndFrames-- + } else if frame.IsBeginFrame() { + numberOfExpectedEndFrames++ + } + } +} + +func (codecUtil) EncodeNullable(message *proto.ClientMessage, value interface{}, encoder Encoder) { + if value == nil { + message.AddFrame(proto.NullFrame.Copy()) + } else { + encoder(message, value) + } +} + +func (codecUtil) EncodeNullableForString(message *proto.ClientMessage, value string) { + if strings.TrimSpace(value) == "" { + message.AddFrame(proto.NullFrame.Copy()) + } else { + EncodeString(message, value) + } +} + +func (codecUtil) EncodeNullableForBitmapIndexOptions(message *proto.ClientMessage, options *types.BitmapIndexOptions) { + if options == nil { + message.AddFrame(proto.NullFrame.Copy()) + } else { + EncodeBitmapIndexOptions(message, *options) + } +} + +func (codecUtil) EncodeNullableForData(message *proto.ClientMessage, data *iserialization.Data) { + if data == nil { + message.AddFrame(proto.NullFrame.Copy()) + } else { + EncodeData(message, data) + } +} + +func (c codecUtil) DecodeNullableForData(frameIterator *proto.ForwardFrameIterator) *iserialization.Data { + if c.NextFrameIsNullFrame(frameIterator) { + return nil + } + return DecodeData(frameIterator) +} + +func (c codecUtil) DecodeNullableForAddress(frameIterator *proto.ForwardFrameIterator) *pubcluster.Address { + if c.NextFrameIsNullFrame(frameIterator) { + return nil + } + addr := DecodeAddress(frameIterator) + return &addr +} + +func (c codecUtil) DecodeNullableForLongArray(frameIterator *proto.ForwardFrameIterator) []int64 { + if c.NextFrameIsNullFrame(frameIterator) { + return nil + } + return DecodeLongArray(frameIterator) +} + +func (c codecUtil) DecodeNullableForString(frameIterator *proto.ForwardFrameIterator) string { + if c.NextFrameIsNullFrame(frameIterator) { + return "" + } + return DecodeString(frameIterator) +} + +func (codecUtil) NextFrameIsDataStructureEndFrame(frameIterator *proto.ForwardFrameIterator) bool { + return frameIterator.PeekNext().IsEndFrame() +} + +func (codecUtil) NextFrameIsNullFrame(frameIterator *proto.ForwardFrameIterator) bool { + isNullFrame := frameIterator.PeekNext().IsNullFrame() + if isNullFrame { + frameIterator.Next() + } + return isNullFrame +} + +func (c codecUtil) DecodeNullableForBitmapIndexOptions(frameIterator *proto.ForwardFrameIterator) types.BitmapIndexOptions { + if c.NextFrameIsNullFrame(frameIterator) { + return types.BitmapIndexOptions{} + } + return DecodeBitmapIndexOptions(frameIterator) +} + +func (c codecUtil) DecodeNullableForSimpleEntryView(frameIterator *proto.ForwardFrameIterator) *types.SimpleEntryView { + if c.NextFrameIsNullFrame(frameIterator) { + return nil + } + return DecodeSimpleEntryView(frameIterator) +} + +func EncodeByteArray(message *proto.ClientMessage, value []byte) { + message.AddFrame(proto.NewFrame(value)) +} + +func DecodeByteArray(frameIterator *proto.ForwardFrameIterator) []byte { + return frameIterator.Next().Content +} + +func EncodeData(message *proto.ClientMessage, value interface{}) { + message.AddFrame(proto.NewFrame(value.(*iserialization.Data).ToByteArray())) +} + +func EncodeNullableData(message *proto.ClientMessage, data *iserialization.Data) { + if data == nil { + message.AddFrame(proto.NullFrame.Copy()) + } else { + message.AddFrame(proto.NewFrame(data.ToByteArray())) + } +} + +func DecodeData(frameIterator *proto.ForwardFrameIterator) *iserialization.Data { + return iserialization.NewData(frameIterator.Next().Content) +} + +func DecodeNullableData(frameIterator *proto.ForwardFrameIterator) *iserialization.Data { + if CodecUtil.NextFrameIsNullFrame(frameIterator) { + return nil + } + return DecodeData(frameIterator) +} + +func EncodeEntryList(message *proto.ClientMessage, entries []proto.Pair, keyEncoder, valueEncoder Encoder) { + message.AddFrame(proto.BeginFrame.Copy()) + for _, value := range entries { + keyEncoder(message, value.Key()) + valueEncoder(message, value.Value()) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func EncodeEntryListForStringAndString(message *proto.ClientMessage, entries []proto.Pair) { + message.AddFrame(proto.BeginFrame.Copy()) + for _, value := range entries { + EncodeString(message, value.Key()) + EncodeString(message, value.Value()) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func EncodeEntryListForStringAndByteArray(message *proto.ClientMessage, entries []proto.Pair) { + message.AddFrame(proto.BeginFrame.Copy()) + for _, value := range entries { + EncodeString(message, value.Key()) + EncodeByteArray(message, value.Value().([]byte)) + } + message.AddFrame(proto.EndFrame.Copy()) + +} + +func EncodeEntryListForDataAndData(message *proto.ClientMessage, entries []proto.Pair) { + message.AddFrame(proto.BeginFrame.Copy()) + for _, value := range entries { + EncodeData(message, value.Key()) + EncodeData(message, value.Value()) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func EncodeEntryListForDataAndListData(message *proto.ClientMessage, entries []proto.Pair) { + message.AddFrame(proto.BeginFrame.Copy()) + for _, value := range entries { + EncodeData(message, value.Key()) + EncodeListData(message, value.Value().([]*iserialization.Data)) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func EncodeNullableEntryList(message *proto.ClientMessage, entries []proto.Pair, keyEncoder, valueEncoder Encoder) { + if len(entries) == 0 { + message.AddFrame(proto.NullFrame.Copy()) + } else { + EncodeEntryList(message, entries, keyEncoder, valueEncoder) + } +} + +func DecodeEntryList(frameIterator *proto.ForwardFrameIterator, keyDecoder, valueDecoder Decoder) []proto.Pair { + result := make([]proto.Pair, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + key := keyDecoder(frameIterator) + value := valueDecoder(frameIterator) + result = append(result, proto.NewPair(key, value)) + } + frameIterator.Next() + return result +} + +func DecodeNullableEntryList(frameIterator *proto.ForwardFrameIterator, keyDecoder, valueDecoder Decoder) []proto.Pair { + if CodecUtil.NextFrameIsNullFrame(frameIterator) { + return nil + } + return DecodeEntryList(frameIterator, keyDecoder, valueDecoder) +} + +func DecodeEntryListForStringAndEntryListIntegerLong(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + result := make([]proto.Pair, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + key := DecodeString(frameIterator) + value := DecodeEntryListIntegerLong(frameIterator) + result = append(result, proto.NewPair(key, value)) + } + frameIterator.Next() + return result +} + +func DecodeEntryListForDataAndData(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + result := make([]proto.Pair, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + key := DecodeData(frameIterator) + value := DecodeData(frameIterator) + result = append(result, proto.NewPair(key, value)) + } + frameIterator.Next() + return result +} + +func EncodeListIntegerIntegerInteger(message *proto.ClientMessage, entries []proto.Pair) { + entryCount := len(entries) + frame := proto.NewFrame(make([]byte, entryCount*proto.EntrySizeInBytes)) + for i := 0; i < entryCount; i++ { + FixSizedTypesCodec.EncodeInt(frame.Content, int32(i*proto.EntrySizeInBytes), entries[i].Key().(int32)) + FixSizedTypesCodec.EncodeInt(frame.Content, int32(i*proto.EntrySizeInBytes+proto.IntSizeInBytes), entries[i].Value().(int32)) + } + message.AddFrame(frame) +} + +func DecodeListIntegerIntegerInteger(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + frame := frameIterator.Next() + itemCount := len(frame.Content) / proto.EntrySizeInBytes + result := make([]proto.Pair, itemCount) + for i := 0; i < itemCount; i++ { + key := FixSizedTypesCodec.DecodeInt(frame.Content, int32(i*proto.EntrySizeInBytes)) + value := FixSizedTypesCodec.DecodeInt(frame.Content, int32(i*proto.EntrySizeInBytes+proto.IntSizeInBytes)) + result = append(result, proto.NewPair(key, value)) + } + return result +} + +func EncodeEntryListUUIDLong(message *proto.ClientMessage, entries []proto.Pair) { + size := len(entries) + content := make([]byte, size*proto.EntrySizeInBytes) + newFrame := proto.NewFrame(content) + for i, entry := range entries { + key := entry.Key().(types.UUID) + value := entry.Value().(int64) + FixSizedTypesCodec.EncodeUUID(content, int32(i*proto.EntrySizeInBytes), key) + FixSizedTypesCodec.EncodeLong(content, int32(i*proto.EntrySizeInBytes+proto.UUIDSizeInBytes), value) + } + message.AddFrame(newFrame) +} + +func EncodeEntryListIntegerInteger(message *proto.ClientMessage, entries []proto.Pair) { + size := len(entries) + content := make([]byte, size*proto.EntrySizeInBytes) + newFrame := proto.NewFrame(content) + for i, entry := range entries { + key := entry.Key().(int32) + value := entry.Value().(int32) + FixSizedTypesCodec.EncodeInt(content, int32(i*proto.EntrySizeInBytes), key) + FixSizedTypesCodec.EncodeInt(content, int32(i*proto.EntrySizeInBytes+proto.UUIDSizeInBytes), value) + } + message.AddFrame(newFrame) +} + +func DecodeEntryListUUIDLong(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + nextFrame := frameIterator.Next() + itemCount := len(nextFrame.Content) / proto.EntrySizeInBytes + content := make([]proto.Pair, itemCount) + for i := 0; i < itemCount; i++ { + uuid := FixSizedTypesCodec.DecodeUUID(nextFrame.Content, int32(i*proto.EntrySizeInBytes)) + value := FixSizedTypesCodec.DecodeLong(nextFrame.Content, int32(i*proto.EntrySizeInBytes+proto.UUIDSizeInBytes)) + content[i] = proto.NewPair(uuid, value) + } + return content +} + +func DecodeEntryListIntegerInteger(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + nextFrame := frameIterator.Next() + itemCount := len(nextFrame.Content) / proto.EntrySizeInBytes + content := make([]proto.Pair, itemCount) + for i := 0; i < itemCount; i++ { + key := FixSizedTypesCodec.DecodeInt(nextFrame.Content, int32(i*proto.EntrySizeInBytes)) + value := FixSizedTypesCodec.DecodeInt(nextFrame.Content, int32(i*proto.EntrySizeInBytes+proto.IntSizeInBytes)) + content[i] = proto.NewPair(key, value) + } + return content +} + +func EncodeEntryListUUIDListInteger(message *proto.ClientMessage, entries []proto.Pair) { + entryCount := len(entries) + uuids := make([]types.UUID, entryCount) + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < entryCount; i++ { + entry := entries[i] + key := entry.Key().(types.UUID) + value := entry.Value().([]int32) + uuids[i] = key + EncodeListInteger(message, value) + } + message.AddFrame(proto.NewEndFrame()) + EncodeListUUID(message, uuids) +} + +func DecodeEntryListUUIDListInteger(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + values := DecodeListMultiFrameWithListInteger(frameIterator) + keys := DecodeListUUID(frameIterator) + keySize := len(keys) + result := make([]proto.Pair, keySize) + for i := 0; i < keySize; i++ { + result[i] = proto.NewPair(keys, values) + } + return result +} + +func DecodeEntryListIntegerUUID(frameIterator *proto.ForwardFrameIterator) []proto.Pair { + frame := frameIterator.Next() + entryCount := len(frame.Content) / proto.EntrySizeInBytes + result := make([]proto.Pair, entryCount) + for i := 0; i < entryCount; i++ { + key := FixSizedTypesCodec.DecodeInt(frame.Content, int32(i*proto.EntrySizeInBytes)) + value := FixSizedTypesCodec.DecodeUUID(frame.Content, int32(i*proto.EntrySizeInBytes+proto.IntSizeInBytes)) + result[i] = proto.NewPair(key, value) + } + return result +} + +func DecodeEntryListIntegerLong(iterator *proto.ForwardFrameIterator) []proto.Pair { + frame := iterator.Next() + entryCount := len(frame.Content) / proto.EntrySizeInBytes + result := make([]proto.Pair, entryCount) + for i := 0; i < entryCount; i++ { + key := FixSizedTypesCodec.DecodeInt(frame.Content, int32(i*proto.EntrySizeInBytes)) + value := FixSizedTypesCodec.DecodeLong(frame.Content, int32(i*proto.EntrySizeInBytes+proto.IntSizeInBytes)) + result[i] = proto.NewPair(key, value) + } + return result +} + +// fixSizedTypesCodec +type fixSizedTypesCodec struct{} + +var FixSizedTypesCodec fixSizedTypesCodec + +func (fixSizedTypesCodec) EncodeInt(buffer []byte, offset, value int32) { + binary.LittleEndian.PutUint32(buffer[offset:], uint32(value)) +} + +func (fixSizedTypesCodec) DecodeInt(buffer []byte, offset int32) int32 { + return int32(binary.LittleEndian.Uint32(buffer[offset:])) +} + +func (fixSizedTypesCodec) EncodeLong(buffer []byte, offset int32, value int64) { + binary.LittleEndian.PutUint64(buffer[offset:], uint64(value)) +} + +func (fixSizedTypesCodec) DecodeLong(buffer []byte, offset int32) int64 { + return int64(binary.LittleEndian.Uint64(buffer[offset:])) +} + +func (fixSizedTypesCodec) EncodeBoolean(buffer []byte, offset int32, value bool) { + if value { + buffer[offset] = 1 + } else { + buffer[offset] = 0 + } +} + +func (fixSizedTypesCodec) DecodeBoolean(buffer []byte, offset int32) bool { + return buffer[offset] == 1 +} + +func (fixSizedTypesCodec) EncodeByte(buffer []byte, offset int32, value byte) { + buffer[offset] = value +} + +func (fixSizedTypesCodec) DecodeByte(buffer []byte, offset int32) byte { + return buffer[offset] +} + +func (fixSizedTypesCodec) EncodeUUID(buffer []byte, offset int32, uuid types.UUID) { + isNullEncode := uuid.Default() + FixSizedTypesCodec.EncodeBoolean(buffer, offset, isNullEncode) + if isNullEncode { + return + } + bufferOffset := offset + proto.BooleanSizeInBytes + FixSizedTypesCodec.EncodeLong(buffer, bufferOffset, int64(uuid.MostSignificantBits())) + FixSizedTypesCodec.EncodeLong(buffer, bufferOffset+proto.LongSizeInBytes, int64(uuid.LeastSignificantBits())) +} + +func (fixSizedTypesCodec) DecodeUUID(buffer []byte, offset int32) types.UUID { + isNull := FixSizedTypesCodec.DecodeBoolean(buffer, offset) + if isNull { + return types.UUID{} + } + + mostSignificantOffset := offset + proto.BooleanSizeInBytes + leastSignificantOffset := mostSignificantOffset + proto.LongSizeInBytes + mostSignificant := uint64(FixSizedTypesCodec.DecodeLong(buffer, mostSignificantOffset)) + leastSignificant := uint64(FixSizedTypesCodec.DecodeLong(buffer, leastSignificantOffset)) + + return types.NewUUIDWith(mostSignificant, leastSignificant) +} + +func EncodeListInteger(message *proto.ClientMessage, entries []int32) { + itemCount := len(entries) + content := make([]byte, itemCount*proto.IntSizeInBytes) + newFrame := proto.NewFrame(content) + for i := 0; i < itemCount; i++ { + FixSizedTypesCodec.EncodeInt(newFrame.Content, int32(i*proto.IntSizeInBytes), entries[i]) + } + message.AddFrame(newFrame) +} + +func DecodeListInteger(frameIterator *proto.ForwardFrameIterator) []int32 { + frame := frameIterator.Next() + itemCount := len(frame.Content) / proto.IntSizeInBytes + result := make([]int32, itemCount) + for i := 0; i < itemCount; i++ { + result[i] = FixSizedTypesCodec.DecodeInt(frame.Content, int32(i*proto.IntSizeInBytes)) + } + return result +} + +func EncodeListLong(message *proto.ClientMessage, entries []int64) { + itemCount := len(entries) + frame := proto.NewFrame(make([]byte, itemCount*proto.LongSizeInBytes)) + for i := 0; i < itemCount; i++ { + FixSizedTypesCodec.EncodeLong(frame.Content, int32(i*proto.LongSizeInBytes), entries[i]) + } + message.AddFrame(frame) +} + +func DecodeListLong(frameIterator *proto.ForwardFrameIterator) []int64 { + frame := frameIterator.Next() + itemCount := len(frame.Content) / proto.LongSizeInBytes + result := make([]int64, itemCount) + for i := 0; i < itemCount; i++ { + result[i] = FixSizedTypesCodec.DecodeLong(frame.Content, int32(i*proto.LongSizeInBytes)) + } + return result +} + +func EncodeListMultiFrame(message *proto.ClientMessage, values []*iserialization.Data, encoder Encoder) { + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < len(values); i++ { + encoder(message, values[i]) + } + message.AddFrame(proto.NewEndFrame()) +} + +func EncodeListMultiFrameForData(message *proto.ClientMessage, values []*iserialization.Data) { + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < len(values); i++ { + EncodeData(message, values[i]) + } + message.AddFrame(proto.NewEndFrame()) +} + +func EncodeListMultiFrameForString(message *proto.ClientMessage, values []string) { + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < len(values); i++ { + EncodeString(message, values[i]) + } + message.AddFrame(proto.NewEndFrame()) +} + +func EncodeListMultiFrameForStackTraceElement(message *proto.ClientMessage, values []ihzerrors.StackTraceElement) { + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < len(values); i++ { + EncodeStackTraceElement(message, values[i]) + } + message.AddFrame(proto.NewEndFrame()) +} + +func EncodeListMultiFrameContainsNullable(message *proto.ClientMessage, values []*iserialization.Data, encoder Encoder) { + message.AddFrame(proto.NewBeginFrame()) + for i := 0; i < len(values); i++ { + if values[i] == nil { + message.AddFrame(proto.NullFrame) + } else { + encoder(message, values[i]) + } + } + message.AddFrame(proto.NewEndFrame()) +} + +func EncodeListMultiFrameNullable(message *proto.ClientMessage, values []*iserialization.Data, encoder Encoder) { + if len(values) == 0 { + message.AddFrame(proto.NullFrame) + } else { + EncodeListMultiFrame(message, values, encoder) + } +} + +func DecodeListMultiFrame(frameIterator *proto.ForwardFrameIterator, decoder func(frameIterator *proto.ForwardFrameIterator)) { + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + decoder(frameIterator) + } + frameIterator.Next() +} + +func DecodeListMultiFrameForData(frameIterator *proto.ForwardFrameIterator) []*iserialization.Data { + result := make([]*iserialization.Data, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeData(frameIterator)) + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameWithListInteger(frameIterator *proto.ForwardFrameIterator) []int32 { + result := make([]int32, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeListInteger(frameIterator)...) + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameForMemberInfo(frameIterator *proto.ForwardFrameIterator) []pubcluster.MemberInfo { + result := make([]pubcluster.MemberInfo, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeMemberInfo(frameIterator)) + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameForStackTraceElement(frameIterator *proto.ForwardFrameIterator) []ihzerrors.StackTraceElement { + var result []ihzerrors.StackTraceElement + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeStackTraceElement(frameIterator)) + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameForString(frameIterator *proto.ForwardFrameIterator) []string { + result := make([]string, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeString(frameIterator)) + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameForDataContainsNullable(frameIterator *proto.ForwardFrameIterator) []*iserialization.Data { + result := make([]*iserialization.Data, 0) + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + if CodecUtil.NextFrameIsNullFrame(frameIterator) { + result = append(result, nil) + } else { + result = append(result, DecodeData(frameIterator)) + } + } + frameIterator.Next() + return result +} + +func DecodeListMultiFrameForDistributedObjectInfo(frameIterator *proto.ForwardFrameIterator) []types.DistributedObjectInfo { + var result []types.DistributedObjectInfo + frameIterator.Next() + for !CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) { + result = append(result, DecodeDistributedObjectInfo(frameIterator)) + } + frameIterator.Next() + return result +} + +func DecodeDistributedObjectInfo(frameIterator *proto.ForwardFrameIterator) types.DistributedObjectInfo { + frameIterator.Next() + serviceName := DecodeString(frameIterator) + name := DecodeString(frameIterator) + CodecUtil.FastForwardToEndFrame(frameIterator) + return types.DistributedObjectInfo{Name: name, ServiceName: serviceName} +} + +func EncodeListData(message *proto.ClientMessage, entries []*iserialization.Data) { + EncodeListMultiFrameForData(message, entries) +} + +func DecodeListData(frameIterator *proto.ForwardFrameIterator) []*iserialization.Data { + return DecodeListMultiFrameForData(frameIterator) +} + +func EncodeListUUID(message *proto.ClientMessage, entries []types.UUID) { + itemCount := len(entries) + content := make([]byte, itemCount*proto.UUIDSizeInBytes) + newFrame := proto.NewFrame(content) + for i := 0; i < itemCount; i++ { + FixSizedTypesCodec.EncodeUUID(content, int32(i*proto.UUIDSizeInBytes), entries[i]) + } + message.AddFrame(newFrame) +} + +func DecodeListUUID(frameIterator *proto.ForwardFrameIterator) []types.UUID { + frame := frameIterator.Next() + itemCount := len(frame.Content) / proto.UUIDSizeInBytes + result := make([]types.UUID, itemCount) + for i := 0; i < itemCount; i++ { + result[i] = FixSizedTypesCodec.DecodeUUID(frame.Content, int32(i*proto.UUIDSizeInBytes)) + } + return result +} + +func EncodeLongArray(message *proto.ClientMessage, entries []int64) { + itemCount := len(entries) + frame := proto.NewFrame(make([]byte, itemCount*proto.LongSizeInBytes)) + for i := 0; i < itemCount; i++ { + FixSizedTypesCodec.EncodeLong(frame.Content, int32(i*proto.LongSizeInBytes), entries[i]) + } + message.AddFrame(frame) +} + +func DecodeLongArray(frameIterator *proto.ForwardFrameIterator) []int64 { + frame := frameIterator.Next() + itemCount := len(frame.Content) / proto.LongSizeInBytes + result := make([]int64, itemCount) + for i := 0; i < itemCount; i++ { + result[i] = FixSizedTypesCodec.DecodeLong(frame.Content, int32(i*proto.LongSizeInBytes)) + } + return result +} + +func EncodeMapForStringAndString(message *proto.ClientMessage, values map[string]string) { + message.AddFrame(proto.BeginFrame.Copy()) + for key, value := range values { + EncodeString(message, key) + EncodeString(message, value) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func EncodeMapForEndpointQualifierAndAddress(message *proto.ClientMessage, values map[pubcluster.EndpointQualifier]pubcluster.Address) { + message.AddFrame(proto.BeginFrame.Copy()) + for key, value := range values { + EncodeEndpointQualifier(message, key) + EncodeAddress(message, value) + } + message.AddFrame(proto.EndFrame.Copy()) +} + +func DecodeMapForStringAndString(iterator *proto.ForwardFrameIterator) map[string]string { + result := map[string]string{} + iterator.Next() + for !iterator.PeekNext().IsEndFrame() { + key := DecodeString(iterator) + value := DecodeString(iterator) + result[key] = value + } + iterator.Next() + return result +} + +func DecodeMapForEndpointQualifierAndAddress(iterator *proto.ForwardFrameIterator) interface{} { + result := map[pubcluster.EndpointQualifier]pubcluster.Address{} + iterator.Next() + for !iterator.PeekNext().IsEndFrame() { + key := DecodeEndpointQualifier(iterator) + value := DecodeAddress(iterator) + result[key] = value + } + iterator.Next() + return result +} + +func EncodeString(message *proto.ClientMessage, value interface{}) { + message.AddFrame(proto.NewFrame([]byte(value.(string)))) +} + +func DecodeString(frameIterator *proto.ForwardFrameIterator) string { + return string(frameIterator.Next().Content) +} + +func DecodeError(msg *proto.ClientMessage) *ihzerrors.ServerError { + frameIterator := msg.FrameIterator() + frameIterator.Next() + errorHolders := []proto.ErrorHolder{} + DecodeListMultiFrame(frameIterator, func(it *proto.ForwardFrameIterator) { + errorHolders = append(errorHolders, DecodeErrorHolder(frameIterator)) + }) + if len(errorHolders) == 0 { + return nil + } + holder := errorHolders[0] + return ihzerrors.NewServerError(holder.ErrorCode, holder.ClassName, holder.Message, holder.StackTraceElements) +} + +func NewEndpointQualifier(qualifierType int32, identifier string) pubcluster.EndpointQualifier { + return pubcluster.EndpointQualifier{ + Type: pubcluster.EndpointQualifierType(qualifierType), + Identifier: identifier, + } +} + +// DistributedObject is the base interface for all distributed objects. +type DistributedObject interface { + // Destroy destroys this object cluster-wide. + // Destroy clears and releases all resources for this object. + Destroy() (bool, error) + + // Name returns the unique name for this DistributedObject. + Name() string + + // PartitionKey returns the key of partition this DistributedObject is assigned to. The returned value only has meaning + // for a non partitioned data structure like an IAtomicLong. For a partitioned data structure like an Map + // the returned value will not be nil, but otherwise undefined. + PartitionKey() string + + // ServiceName returns the service name for this object. + ServiceName() string +} + +func NewMemberVersion(major, minor, patch byte) pubcluster.MemberVersion { + return pubcluster.MemberVersion{Major: major, Minor: minor, Patch: patch} +} + +func NewMemberInfo( + address pubcluster.Address, + uuid types.UUID, + attributes map[string]string, + liteMember bool, + version pubcluster.MemberVersion, + addressMapExists bool, + addressMap interface{}) pubcluster.MemberInfo { + var addrMap map[pubcluster.EndpointQualifier]pubcluster.Address + if addressMapExists { + addrMap = addressMap.(map[pubcluster.EndpointQualifier]pubcluster.Address) + } else { + addrMap = map[pubcluster.EndpointQualifier]pubcluster.Address{} + } + return pubcluster.MemberInfo{ + Address: address, + UUID: uuid, + Attributes: attributes, + LiteMember: liteMember, + Version: version, + AddressMap: addrMap, + } +} + +func EncodeAddress(clientMessage *proto.ClientMessage, address pubcluster.Address) { + host, portStr, err := net.SplitHostPort(address.String()) + if err != nil { + panic(fmt.Errorf("parsing address: %w", err)) + } + port, err := strconv.Atoi(portStr) + if err != nil { + panic(fmt.Errorf("parsing address: %w", err)) + } + clientMessage.AddFrame(proto.BeginFrame.Copy()) + initialFrame := proto.NewFrame(make([]byte, AddressCodecPortInitialFrameSize)) + FixSizedTypesCodec.EncodeInt(initialFrame.Content, AddressCodecPortFieldOffset, int32(port)) + clientMessage.AddFrame(initialFrame) + EncodeString(clientMessage, host) + clientMessage.AddFrame(proto.EndFrame.Copy()) +} diff --git a/go/verbatim/builtin_test.go b/go/verbatim/builtin_test.go new file mode 100644 index 000000000..351fcd0bd --- /dev/null +++ b/go/verbatim/builtin_test.go @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package codec + +import ( + "encoding/binary" + "testing" + + "github.com/hazelcast/hazelcast-go-client/types" + + "github.com/stretchr/testify/assert" + + "github.com/hazelcast/hazelcast-go-client/internal/proto" +) + +func TestCodecUtil_FastForwardToEndFrame(t *testing.T) { + // given + frame1 := proto.NewFrameWith([]byte("value-1"), proto.BeginDataStructureFlag) + frame2 := proto.NewFrameWith([]byte("value-2"), proto.EndDataStructureFlag) + frame3 := proto.NewFrameWith([]byte("value-3"), proto.EndDataStructureFlag) + + message := proto.NewClientMessage(frame1) + message.AddFrame(frame2) + message.AddFrame(frame3) + + //when + iterator := message.FrameIterator() + CodecUtil.FastForwardToEndFrame(iterator) + + //then + assert.False(t, iterator.HasNext()) +} + +func TestCodecUtil_EncodeNullable(t *testing.T) { + //given + frame1 := proto.NewFrame([]byte("value-0")) + message := proto.NewClientMessage(frame1) + + //when + CodecUtil.EncodeNullable(message, "encode-value-1", EncodeString) + + //then + iterator := message.FrameIterator() + assert.Equal(t, string(iterator.Next().Content), "value-0") + assert.Equal(t, string(iterator.Next().Content), "encode-value-1") + assert.Equal(t, string(message.Frames[len(message.Frames)-1].Content), "encode-value-1") +} + +func TestCodecUtil_NextFrameIsDataStructureEndFrame(t *testing.T) { + //given + byteValue := []byte("value-0") + flags := uint16(proto.EndDataStructureFlag) + frame := proto.NewFrameWith(byteValue, flags) + message := proto.NewClientMessage(frame) + //when + frameIterator := message.FrameIterator() + isEndFrame := CodecUtil.NextFrameIsDataStructureEndFrame(frameIterator) + + //then + assert.True(t, isEndFrame) +} + +func TestCodecUtil_NextFrameIsNullFrame(t *testing.T) { + //given + nullFrame := proto.NullFrame + message := proto.NewClientMessage(nullFrame) + frame := proto.NewFrame([]byte("value-0")) + message.AddFrame(frame) + frameIterator := message.FrameIterator() + + //when + isNextFrameIsNull := CodecUtil.NextFrameIsNullFrame(frameIterator) + + //then + assert.True(t, isNextFrameIsNull) +} + +func TestCodecUtil_NextFrameIsNullFrame_Return_False_When_Next_Frame_Is_Not_Null(t *testing.T) { + //given + frame1 := proto.NewFrame([]byte("value-1")) + frame2 := proto.NewFrame([]byte("value-2")) + message := proto.NewClientMessage(frame1) + message.AddFrame(frame2) + frameIterator := message.FrameIterator() + + //when + isNextFrameIsNull := CodecUtil.NextFrameIsNullFrame(frameIterator) + + //then + assert.False(t, isNextFrameIsNull) +} + +func TestByteArrayCodec_Encode(t *testing.T) { + //given + value := []byte("value-1") + message := proto.NewClientMessageForEncode() + + //when + EncodeByteArray(message, value) + + //then + iterator := message.FrameIterator() + assert.Equal(t, string(iterator.Next().Content), "value-1") +} + +func TestByteArrayCodec_Decode(t *testing.T) { + //given + value := []byte("value-1") + message := proto.NewClientMessage(proto.NewFrame(value)) + + //when + decode := DecodeByteArray(message.FrameIterator()) + + //then + assert.Equal(t, string(decode), "value-1") +} + +func TestCodecUtil_EncodeNullable_If_Value_Is_Null_Add_Null_Frame_To_Message(t *testing.T) { + //given + frame1 := proto.NewFrame([]byte("value-0")) + message := proto.NewClientMessage(frame1) + + //when + CodecUtil.EncodeNullable(message, nil, EncodeString) + + //then + iterator := message.FrameIterator() + assert.Equal(t, string(iterator.Next().Content), "value-0") + assert.True(t, iterator.Next().IsNullFrame()) +} + +func TestDataCodec_EncodeNullable_When_Data_Is_Nil(t *testing.T) { + //given + message := proto.NewClientMessageForEncode() + + //when + EncodeNullableData(message, nil) + + //then + iterator := message.FrameIterator() + assert.Len(t, iterator.Next().Content, 0) +} + +func TestDataCodec_Decode(t *testing.T) { + //given + bytes := []byte("value-0") + frame := proto.NewFrame(bytes) + message := proto.NewClientMessageForDecode(frame) + frameIterator := message.FrameIterator() + + //when + decode := DecodeNullableData(frameIterator) + + //then + assert.Equal(t, decode.ToByteArray(), bytes) +} + +func TestDataCodec_Decode_When_Data_Is_Nil(t *testing.T) { + //given + nullFrame := proto.NullFrame + message := proto.NewClientMessage(nullFrame) + frameIterator := message.FrameIterator() + + //when + decode := DecodeNullableData(frameIterator) + + //then + assert.Nil(t, decode) +} + +func TestEntryListCodec_Encode(t *testing.T) { + //given + bytes := []byte("value-0") + frame := proto.NewFrame(bytes) + message := proto.NewClientMessageForDecode(frame) + pairs := make([]proto.Pair, 0) + pairs = append(pairs, proto.NewPair("key", "value")) + + //when + EncodeEntryList(message, pairs, EncodeString, EncodeString) + + //then + frameIterator := message.FrameIterator() + assert.Equal(t, string(frameIterator.Next().Content), "value-0") + + beginFrame := frameIterator.Next() + assert.Empty(t, beginFrame.Content) + assert.True(t, beginFrame.IsBeginFrame()) + + assert.Equal(t, string(frameIterator.Next().Content), "key") + assert.Equal(t, string(frameIterator.Next().Content), "value") + + endFrame := frameIterator.Next() + assert.Empty(t, endFrame.Content) + assert.True(t, endFrame.IsEndFrame()) +} + +func TestEntryListCodec_Encode_When_Entries_Is_Empty(t *testing.T) { + //given + bytes := []byte("value-0") + frame := proto.NewFrame(bytes) + message := proto.NewClientMessageForDecode(frame) + pairs := make([]proto.Pair, 0) + + //when + EncodeEntryList(message, pairs, EncodeString, EncodeString) + + //then + frameIterator := message.FrameIterator() + assert.Equal(t, string(frameIterator.Next().Content), "value-0") + + beginFrame := frameIterator.Next() + assert.Empty(t, beginFrame.Content) + assert.True(t, beginFrame.IsBeginFrame()) + + endFrame := frameIterator.Next() + assert.Empty(t, endFrame.Content) + assert.True(t, endFrame.IsEndFrame()) +} + +func TestEntryListCodec_EncodeNullable(t *testing.T) { + //given + bytes := []byte("value-0") + frame := proto.NewFrame(bytes) + message := proto.NewClientMessageForDecode(frame) + pairs := make([]proto.Pair, 0) + pairs = append(pairs, proto.NewPair("key", "value")) + + //when + EncodeNullableEntryList(message, pairs, EncodeString, EncodeString) + + //then + frameIterator := message.FrameIterator() + assert.Equal(t, string(frameIterator.Next().Content), "value-0") + + beginFrame := frameIterator.Next() + assert.Empty(t, beginFrame.Content) + assert.True(t, beginFrame.IsBeginFrame()) + + assert.Equal(t, string(frameIterator.Next().Content), "key") + assert.Equal(t, string(frameIterator.Next().Content), "value") + + endFrame := frameIterator.Next() + assert.Empty(t, endFrame.Content) + assert.True(t, endFrame.IsEndFrame()) +} + +func TestEntryListCodec_EncodeNullable_When_Entries_Is_Empty(t *testing.T) { + //given + message := proto.NewClientMessageForEncode() + pairs := make([]proto.Pair, 0) + + //when + EncodeNullableEntryList(message, pairs, EncodeString, EncodeString) + + //then + frameIterator := message.FrameIterator() + assert.True(t, frameIterator.Next().IsNullFrame()) +} + +func TestEntryListCodec_DecodeNullable(t *testing.T) { + //given + nullFrame := proto.NullFrame + message := proto.NewClientMessageForDecode(nullFrame) + iterator := message.FrameIterator() + + //when + results := DecodeNullableEntryList(iterator, DecodeData, DecodeData) + + //then + assert.Nil(t, results) +} + +func TestEntryListCodec_DecodeNullable_When_Next_Frame_Is_Null_Frame(t *testing.T) { + //given + message := proto.NewClientMessageForDecode(proto.NullFrame) + iterator := message.FrameIterator() + + //when + results := DecodeNullableEntryList(iterator, DecodeData, DecodeData) + + //then + assert.Empty(t, results) +} + +func TestFixSizedTypesCodec_EncodeInt(t *testing.T) { + //given + buffer := make([]byte, 4) + offset := int32(0) + value := int32(100) + + //when + FixSizedTypesCodec.EncodeInt(buffer, offset, value) + + //then + assert.Equal(t, binary.LittleEndian.Uint32(buffer), uint32(100)) +} + +func TestFixSizedTypesCodec_DecodeInt(t *testing.T) { + //given + buffer := make([]byte, 4) + offset := int32(0) + value := int32(100) + FixSizedTypesCodec.EncodeInt(buffer, offset, value) + + //when + decodeInt := FixSizedTypesCodec.DecodeInt(buffer, offset) + + //then + assert.Equal(t, decodeInt, value) +} + +func TestFixSizedTypesCodec_EncodeLong(t *testing.T) { + //given + buffer := make([]byte, 8) + offset := int32(0) + value := int64(100000000000000000) + + //when + FixSizedTypesCodec.EncodeLong(buffer, offset, value) + + //then + assert.Equal(t, int64(binary.LittleEndian.Uint64(buffer[offset:])), value) +} + +func TestFixSizedTypesCodec_DecodeLong(t *testing.T) { + //given + buffer := make([]byte, 8) + offset := int32(0) + value := int64(100000000000000000) + FixSizedTypesCodec.EncodeLong(buffer, offset, value) + + //when + decodeLong := FixSizedTypesCodec.DecodeLong(buffer, offset) + + //then + assert.Equal(t, decodeLong, value) +} + +func TestFixSizedTypesCodec_EncodeBool(t *testing.T) { + //given + buffer := make([]byte, 1) + offset := int32(0) + + //when + FixSizedTypesCodec.EncodeBoolean(buffer, offset, true) + + //then + assert.True(t, buffer[offset] == 1) +} + +func TestFixSizedTypesCodec_EncodeBool_When_Value_Is_False(t *testing.T) { + //given + buffer := make([]byte, 1) + offset := int32(0) + + //when + FixSizedTypesCodec.EncodeBoolean(buffer, offset, false) + + //then + assert.True(t, buffer[offset] == 0) +} + +func TestFixSizedTypesCodec_EncodeByte(t *testing.T) { + //given + buffer := make([]byte, 1) + offset := int32(0) + + //when + FixSizedTypesCodec.EncodeByte(buffer, offset, 'b') + + //then + assert.Equal(t, string(buffer[0]), "b") +} + +func TestFixSizedTypesCodec_DecodeByte(t *testing.T) { + //given + buffer := make([]byte, 1) + offset := int32(0) + FixSizedTypesCodec.EncodeByte(buffer, offset, 'b') + + //when + decodeByte := FixSizedTypesCodec.DecodeByte(buffer, offset) + + //then + assert.Equal(t, string(decodeByte), "b") +} + +func TestFixSizedTypesCodec_EncodeUUID(t *testing.T) { + //given + buffer := make([]byte, proto.UUIDSizeInBytes) + offset := int32(0) + uuid := types.NewUUID() + + //when + FixSizedTypesCodec.EncodeUUID(buffer, offset, uuid) + + //then + assert.Equal(t, FixSizedTypesCodec.DecodeBoolean(buffer, offset), false) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(buffer, offset+proto.BooleanSizeInBytes), int64(uuid.MostSignificantBits())) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(buffer, offset+proto.BooleanSizeInBytes+proto.LongSizeInBytes), int64(uuid.LeastSignificantBits())) +} + +func TestFixSizedTypesCodec_EncodeUUID_When_UUID_Is_Nil(t *testing.T) { + //given + buffer := make([]byte, proto.UUIDSizeInBytes) + offset := int32(0) + + //when + FixSizedTypesCodec.EncodeUUID(buffer, offset, types.UUID{}) + + //then + assert.Equal(t, FixSizedTypesCodec.DecodeBoolean(buffer, offset), true) +} + +func TestFixSizedTypesCodec_DecodeUUID(t *testing.T) { + //given + buffer := make([]byte, proto.UUIDSizeInBytes) + offset := int32(0) + uuid := types.NewUUID() + FixSizedTypesCodec.EncodeUUID(buffer, offset, uuid) + + //when + decodeUUID := FixSizedTypesCodec.DecodeUUID(buffer, offset) + + //then + assert.Equal(t, FixSizedTypesCodec.DecodeBoolean(buffer, offset), false) + assert.Equal(t, uuid, decodeUUID) + assert.Equal(t, uuid.String(), decodeUUID.String()) + assert.Equal(t, uuid.MostSignificantBits(), decodeUUID.MostSignificantBits()) + assert.Equal(t, uuid.LeastSignificantBits(), decodeUUID.LeastSignificantBits()) +} + +func TestEntryListUUIDLongCodec_Encode(t *testing.T) { + // given + message := proto.NewClientMessageForEncode() + key := types.NewUUID() + value := int64(100) + pairs := make([]proto.Pair, 0) + pairs = append(pairs, proto.NewPair(key, value)) + EncodeEntryListUUIDLong(message, pairs) + + // when + pairs = DecodeEntryListUUIDLong(message.FrameIterator()) + + // then + frame := pairs[0] + assert.Equal(t, frame.Key().(types.UUID).String(), key.String()) + assert.Equal(t, frame.Value().(int64), value) +} + +func TestListUUIDCodec_Encode(t *testing.T) { + // given + message := proto.NewClientMessageForEncode() + entries := make([]types.UUID, 0) + value1 := types.NewUUID() + value2 := types.NewUUID() + entries = append(entries, value1, value2) + + // when + EncodeListUUID(message, entries) + + // then + frame := message.FrameIterator().Next() + decodeUUID1 := FixSizedTypesCodec.DecodeUUID(frame.Content, 0) + assert.Equal(t, value1.String(), decodeUUID1.String()) + decodeUUID2 := FixSizedTypesCodec.DecodeUUID(frame.Content, 17) + assert.Equal(t, value2.String(), decodeUUID2.String()) +} + +func TestListIntegerCodec_Encode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + entries := make([]int32, 0) + entries = append(entries, 1, 2, 3) + + // when + EncodeListInteger(clientMessage, entries) + + // then + frame := clientMessage.FrameIterator().Next() + assert.Equal(t, FixSizedTypesCodec.DecodeInt(frame.Content, 0), int32(1)) + assert.Equal(t, FixSizedTypesCodec.DecodeInt(frame.Content, 4), int32(2)) + assert.Equal(t, FixSizedTypesCodec.DecodeInt(frame.Content, 8), int32(3)) +} + +func TestListIntegerCodec_Decode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + entries := make([]int32, 0) + entries = append(entries, 1, 2, 3) + EncodeListInteger(clientMessage, entries) + + // when + decodeEntries := DecodeListInteger(clientMessage.FrameIterator()) + + // then + assert.Equal(t, decodeEntries[0], int32(1)) + assert.Equal(t, decodeEntries[1], int32(2)) + assert.Equal(t, decodeEntries[2], int32(3)) +} + +func TestListLongCodec_Encode(t *testing.T) { + // given + message := proto.NewClientMessageForEncode() + entries := make([]int64, 0) + entries = append(entries, 1, 2, 3) + + // when + EncodeListLong(message, entries) + + // then + frame := message.FrameIterator().Next() + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 0), int64(1)) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 8), int64(2)) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 16), int64(3)) +} + +func TestListLongCodec_Decode(t *testing.T) { + // given + message := proto.NewClientMessageForEncode() + entries := make([]int64, 0) + entries = append(entries, 1, 2, 3) + EncodeListLong(message, entries) + + // when + result := DecodeListLong(message.FrameIterator()) + + // then + assert.Equal(t, result, entries) +} + +func TestEntryListUUIDListIntegerCodec_Encode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + key := types.NewUUID() + value := make([]int32, 0) + value = append(value, 1, 2, 3) + pair := proto.NewPair(key, value) + entries := make([]proto.Pair, 0) + entries = append(entries, pair) + + // when + EncodeEntryListUUIDListInteger(clientMessage, entries) + + // then + iterator := clientMessage.FrameIterator() + assert.Equal(t, iterator.Next().IsBeginFrame(), true) + integerValues := iterator.Next() + assert.Equal(t, FixSizedTypesCodec.DecodeInt(integerValues.Content, 0), int32(1)) + assert.Equal(t, FixSizedTypesCodec.DecodeInt(integerValues.Content, 4), int32(2)) + assert.Equal(t, FixSizedTypesCodec.DecodeInt(integerValues.Content, 8), int32(3)) + assert.Equal(t, iterator.Next().IsEndFrame(), true) + uuid := FixSizedTypesCodec.DecodeUUID(iterator.Next().Content, 0) + assert.Equal(t, uuid.String(), key.String()) +} + +func TestEntryListUUIDListIntegerCodec_Decode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + key := types.NewUUID() + value := make([]int32, 0) + value = append(value, 1, 2, 3) + pair := proto.NewPair(key, value) + entries := make([]proto.Pair, 0) + entries = append(entries, pair) + EncodeEntryListUUIDListInteger(clientMessage, entries) + + // when + result := DecodeEntryListUUIDListInteger(clientMessage.FrameIterator()) + + // then + assert.Equal(t, len(result), 1) + assert.Equal(t, result[0].Key().([]types.UUID)[0].String(), key.String()) + assert.EqualValues(t, result[0].Value().([]int32), value) +} + +func TestLongArrayCodec_Encode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + entries := make([]int64, 0) + entries = append(entries, 1, 2, 3) + + // when + EncodeLongArray(clientMessage, entries) + + // then + frame := clientMessage.FrameIterator().Next() + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 0), int64(1)) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 8), int64(2)) + assert.Equal(t, FixSizedTypesCodec.DecodeLong(frame.Content, 16), int64(3)) +} + +func TestLongArrayCodec_Decode(t *testing.T) { + // given + clientMessage := proto.NewClientMessageForEncode() + entries := make([]int64, 0) + entries = append(entries, 1, 2, 3) + EncodeLongArray(clientMessage, entries) + + // when + result := DecodeLongArray(clientMessage.FrameIterator()) + + // then + assert.Equal(t, result[0], int64(1)) + assert.Equal(t, result[1], int64(2)) + assert.Equal(t, result[2], int64(3)) +} + +func TestStringCodec_Encode(t *testing.T) { + //given + + value := "value-encode" + frame := proto.NewFrame([]byte("")) + clientMessage := proto.NewClientMessage(frame) + + //when + EncodeString(clientMessage, value) + + //then + content := clientMessage.Frames[len(clientMessage.Frames)-1].Content + assert.Equal(t, value, string(content)) +} diff --git a/requirements.txt b/requirements.txt index 36158e3f3..873495aa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ PyYAML jsonschema -jinja2 \ No newline at end of file +jinja2 +setuptools==72.1.0 \ No newline at end of file diff --git a/util.py b/util.py index 1172ba52f..19ffe2b91 100644 --- a/util.py +++ b/util.py @@ -3,11 +3,12 @@ import json import re import fnmatch -from enum import Enum import os from os import listdir, makedirs from os.path import dirname, isfile, join, realpath from datetime import date +from enum import Enum +from distutils.dir_util import copy_tree import jsonschema import yaml @@ -41,8 +42,6 @@ py_param_name, py_types_encode_decode, py_custom_type_name, - py_decoder_requires_to_object_fn, - py_to_object_fn_in_decode, ) from ts import ( ts_escape_keyword, @@ -56,7 +55,7 @@ MINOR_VERSION_MULTIPLIER = 100 PATCH_VERSION_MULTIPLIER = 1 -ID_VALIDATOR_IGNORE_SET = {"Jet", "Experimental"} +ID_VALIDATOR_IGNORE_SET = {"Jet"} def java_name(type_name): @@ -423,7 +422,7 @@ def validate_services(services, schema_path, no_id_check, protocol_versions): valid = False response_params = method["response"].get("params", []) if contains_invalid_nullability_field(service_method_name, request_params): - valid = False + valid = False return valid @@ -576,13 +575,25 @@ def get_protocol_versions(protocol_defs, custom_codec_defs): return map(str, protocol_versions) +def copy_verbatim_files(output_dir, lang): + cur_dir = os.path.dirname(os.path.realpath(__file__)) + lang_dir = os.path.join(cur_dir, lang) + verbatim_dir = os.path.join(lang_dir, "verbatim") + if not os.path.exists(verbatim_dir): + return + # iterate the verbatim directory and copy files to output + # creating directories if necessary + print("Verbatim files found, copying from %s to %s" % (verbatim_dir, output_dir)) + copy_tree(verbatim_dir, output_dir) + + class SupportedLanguages(Enum): JAVA = "java" CPP = "cpp" CS = "cs" PY = "py" TS = "ts" - # GO = 'go' + GO = 'go' MD = "md" @@ -592,7 +603,7 @@ class SupportedLanguages(Enum): SupportedLanguages.CS: "src/Hazelcast.Net/Protocol/Codecs/", SupportedLanguages.PY: "hazelcast/protocol/codec/", SupportedLanguages.TS: "src/codec/", - # SupportedLanguages.GO: 'internal/proto/' + SupportedLanguages.GO: "hazelcast/protocol/codec/", SupportedLanguages.MD: "documentation", } @@ -602,7 +613,7 @@ class SupportedLanguages(Enum): SupportedLanguages.CS: "src/Hazelcast.Net/Protocol/CustomCodecs/", SupportedLanguages.PY: "hazelcast/protocol/codec/custom/", SupportedLanguages.TS: "src/codec/custom", - # SupportedLanguages.GO: 'internal/proto/' + SupportedLanguages.GO: "hazelcast/protocol/codec", } @@ -615,66 +626,93 @@ def inner(*names): def _snake_cased_name_generator(extension): def inner(*names): - return "%s_codec.%s" % ("_".join([py_param_name(name, False) for name in names]), extension) + return "%s_codec.%s" % ("_".join(map(py_param_name, names)), extension) return inner file_name_generators = { - SupportedLanguages.JAVA: _capitalized_name_generator("java"), - SupportedLanguages.CPP: _snake_cased_name_generator("cpp"), - SupportedLanguages.CS: _capitalized_name_generator("cs"), - SupportedLanguages.PY: _snake_cased_name_generator("py"), - SupportedLanguages.TS: _capitalized_name_generator("ts"), - # SupportedLanguages.GO: 'go' + SupportedLanguages.JAVA: _capitalized_name_generator('java'), + SupportedLanguages.CPP: _snake_cased_name_generator('cpp'), + SupportedLanguages.CS: _capitalized_name_generator('cs'), + SupportedLanguages.PY: _snake_cased_name_generator('py'), + SupportedLanguages.TS: _capitalized_name_generator('ts'), SupportedLanguages.MD: "md", } language_specific_funcs = { - SupportedLanguages.JAVA: { - "lang_types_encode": java_types_encode, - "lang_types_decode": java_types_decode, - "lang_name": java_name, - "param_name": param_name, + "lang_types_encode": { + SupportedLanguages.JAVA: java_types_encode, + SupportedLanguages.CS: cs_types_encode, + SupportedLanguages.CPP: cpp_types_encode, + SupportedLanguages.TS: ts_types_encode, + SupportedLanguages.PY: py_types_encode_decode, + SupportedLanguages.MD: lambda x: x, }, - SupportedLanguages.CS: { - "lang_types_encode": cs_types_encode, - "lang_types_decode": cs_types_decode, - "lang_name": cs_name, - "param_name": param_name, - "escape_keyword": cs_escape_keyword, - "custom_codec_param_name": cs_custom_codec_param_name, - "cs_sizeof": cs_sizeof, - "cs_param_prefix": cs_param_prefix, + "lang_types_decode": { + SupportedLanguages.JAVA: java_types_decode, + SupportedLanguages.CS: cs_types_decode, + SupportedLanguages.CPP: cpp_types_decode, + SupportedLanguages.TS: ts_types_decode, + SupportedLanguages.PY: py_types_encode_decode, + SupportedLanguages.MD: lambda x: x, }, - SupportedLanguages.CPP: { - "lang_types_encode": cpp_types_encode, - "lang_types_decode": cpp_types_decode, - "lang_name": cpp_name, - "param_name": cpp_param_name, + "lang_name": { + SupportedLanguages.JAVA: java_name, + SupportedLanguages.CS: cs_name, + SupportedLanguages.CPP: cpp_name, + SupportedLanguages.TS: java_name, + SupportedLanguages.PY: java_name, + SupportedLanguages.MD: lambda x: x, }, - SupportedLanguages.TS: { - "lang_types_encode": ts_types_encode, - "lang_types_decode": ts_types_decode, - "lang_name": java_name, - "param_name": param_name, - "escape_keyword": ts_escape_keyword, - "get_import_path_holders": ts_get_import_path_holders, + "param_name": { + SupportedLanguages.JAVA: param_name, + SupportedLanguages.CS: param_name, + SupportedLanguages.CPP: cpp_param_name, + SupportedLanguages.TS: param_name, + SupportedLanguages.PY: py_param_name, + SupportedLanguages.MD: lambda x: x, }, - SupportedLanguages.PY: { - "lang_types_encode": py_types_encode_decode, - "lang_types_decode": py_types_encode_decode, - "lang_name": java_name, - "param_name": py_param_name, - "escape_keyword": py_escape_keyword, - "get_import_path_holders": py_get_import_path_holders, - "custom_type_name": py_custom_type_name, - "decoder_requires_to_object_fn": py_decoder_requires_to_object_fn, - "to_object_fn_in_decode": py_to_object_fn_in_decode, + "escape_keyword": { + SupportedLanguages.JAVA: lambda x: x, + SupportedLanguages.CS: cs_escape_keyword, + SupportedLanguages.CPP: lambda x: x, + SupportedLanguages.TS: ts_escape_keyword, + SupportedLanguages.PY: py_escape_keyword, + SupportedLanguages.MD: lambda x: x, }, - SupportedLanguages.MD: { - "param_name": param_name, - } + "get_import_path_holders": { + SupportedLanguages.JAVA: lambda x: x, + SupportedLanguages.CS: lambda x: x, + SupportedLanguages.CPP: lambda x: x, + SupportedLanguages.TS: ts_get_import_path_holders, + SupportedLanguages.PY: py_get_import_path_holders, + SupportedLanguages.MD: lambda x: x, + }, + "custom_type_name": { + SupportedLanguages.JAVA: lambda x: x, + SupportedLanguages.CS: lambda x: x, + SupportedLanguages.CPP: lambda x: x, + SupportedLanguages.TS: lambda x: x, + SupportedLanguages.PY: py_custom_type_name, + SupportedLanguages.MD: lambda x: x, + }, + "custom_codec_param_name": { + SupportedLanguages.JAVA: lambda x,y: y, + SupportedLanguages.CS: cs_custom_codec_param_name, + SupportedLanguages.CPP: lambda x,y: y, + SupportedLanguages.TS: lambda x,y: y, + SupportedLanguages.PY: lambda x,y: y, + SupportedLanguages.MD: lambda x,y: y, + }, + "init_env": { + SupportedLanguages.JAVA: lambda x: x, + # SupportedLanguages.CS: cs_init_env, + SupportedLanguages.CPP: lambda x: x, + SupportedLanguages.TS: lambda x: x, + SupportedLanguages.PY: lambda x: x, + SupportedLanguages.MD: lambda x: x, + } } language_service_ignore_list = { @@ -683,7 +721,6 @@ def inner(*names): SupportedLanguages.CS: cs_ignore_service_list, SupportedLanguages.PY: py_ignore_service_list, SupportedLanguages.TS: ts_ignore_service_list, - # SupportedLanguages.GO: set() } @@ -731,17 +768,24 @@ def create_environment(lang, namespace): env.globals["key_type"] = key_type env.globals["value_type"] = value_type env.globals["namespace"] = namespace + env.globals["lang_types_encode"] = language_specific_funcs["lang_types_encode"][lang] + env.globals["lang_types_decode"] = language_specific_funcs["lang_types_decode"][lang] + env.globals["lang_name"] = language_specific_funcs["lang_name"][lang] + env.globals["param_name"] = language_specific_funcs["param_name"][lang] + env.globals["escape_keyword"] = language_specific_funcs["escape_keyword"][lang] + env.globals["custom_type_name"] = language_specific_funcs["custom_type_name"][lang] + env.globals["custom_codec_param_name"] = language_specific_funcs["custom_codec_param_name"][lang] env.globals["get_size"] = get_size env.globals["is_trivial"] = is_trivial + env.globals["get_import_path_holders"] = language_specific_funcs["get_import_path_holders"][lang] env.globals["copyright_year"] = date.today().year - + try: with os.popen("git rev-parse --short HEAD") as f: env.globals["protocol_commit"] = f.readlines()[0].strip() except: env.globals["protocol_commit"] = "unknown" - for fn_name, fn in language_specific_funcs[lang].items(): - env.globals[fn_name] = fn + env = language_specific_funcs["init_env"][lang](env) return env