From 63416ffa3e2155f367dc1fed832bba017a4484e5 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Tue, 9 May 2017 21:56:30 +0800 Subject: [PATCH 1/9] Add model configuration for machine translation with external memory. --- .../mt_with_external_memory.py | 567 ++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 mt_with_external_memory/mt_with_external_memory.py diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py new file mode 100644 index 0000000000..1f9bcb0eb8 --- /dev/null +++ b/mt_with_external_memory/mt_with_external_memory.py @@ -0,0 +1,567 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.v2 as paddle +import sys +import gzip + +dict_size = 30000 +word_vec_dim = 512 +hidden_size = 1024 +batch_size = 5 +memory_slot_num = 8 +beam_size = 40 +infer_data_num = 3 + + +class ExternalMemory(object): + """ + External neural memory class, with differentiable write/read heads. + + A simplified Neural Turing Machine (NTM) with only content-based + addressing (including content addressing and interpolation, but excluding + convolutional shift and sharpening). It can serve as an external memory + bank, with differential write/read head controllers responsible for storing + and reading information flow dynamically as the model needs. Here, simple + feedforward neural networks are used as the write/read head controllers. + + For more techinical details, please refer to the + `NTM paper `_. + + :param name: Memory name. + :type name: basestring + :param mem_slot_size: Size of memory slot/vector. + :type mem_slot_size: int + :param boot_layer: Boot layer for initializing memory. Sequence layer + with sequence length indicating the number of memory + slots, and size as mem_slot_size. + :type boot_layer: LayerOutput + :param readonly: If true, the memory is read-only, and write function cannot + be called. Default is false. + :type readonly: bool + """ + + def __init__(self, name, mem_slot_size, boot_layer, readonly=False): + self.name = name + self.mem_slot_size = mem_slot_size + self.readonly = readonly + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) + # set memory to constant when readonly=True + if self.readonly: + self.updated_external_memory = paddle.layer.mixed( + name=self.name, + input=[ + paddle.layer.identity_projection(input=self.external_memory) + ], + size=self.mem_slot_size) + + def __content_addressing__(self, key_vector): + """ + Get head's addressing weight via content-based addressing. + """ + # content-based addressing: a=tanh(W*M + U*key) + key_projection = paddle.layer.fc( + input=key_vector, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + key_proj_expanded = paddle.layer.expand( + input=key_projection, expand_as=self.external_memory) + memory_projection = paddle.layer.fc( + input=self.external_memory, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + merged = paddle.layer.addto( + input=[key_proj_expanded, memory_projection], + act=paddle.activation.Tanh()) + # softmax addressing weight: w=softmax(v^T a) + addressing_weight = paddle.layer.fc( + input=merged, + size=1, + act=paddle.activation.SequenceSoftmax(), + bias_attr=False) + return addressing_weight + + def __interpolation__(self, key_vector, addressing_weight): + """ + Interpolate between previous and current addressing weights. + """ + # prepare interpolation scalar gate: g=sigmoid(W*key) + gate = paddle.layer.fc( + input=key_vector, + size=1, + act=paddle.activation.Sigmoid(), + bias_attr=False) + # interpolation: w_t = g*w_t+(1-g)*w_{t-1} + last_addressing_weight = paddle.layer.memory( + name=self.name + "_addressing_weight", size=1, is_seq=True) + gated_addressing_weight = paddle.layer.addto( + name=self.name + "_addressing_weight", + input=[ + last_addressing_weight, + paddle.layer.scaling(weight=gate, input=addressing_weight), + paddle.layer.mixed( + input=paddle.layer.dotmul_operator( + a=gate, b=last_addressing_weight, scale=-1.0), + size=1) + ], + act=paddle.activation.Tanh()) + return gated_addressing_weight + + def __get_addressing_weight__(self, key_vector): + """ + Get final addressing weight for read/write heads, including content + addressing and interpolation. + """ + # current content-based addressing + addressing_weight = self.__content_addressing__(key_vector) + return addressing_weight + # interpolation with previous addresing weight + return self.__interpolation__(key_vector, addressing_weight) + + def write(self, write_key): + """ + Write head for external memory. + + :param write_key: Key vector for write head to generate writing + content and addressing signals. + :type write_key: LayerOutput + """ + # check readonly + if self.readonly: + raise ValueError("ExternalMemory with readonly=True cannot write.") + # get addressing weight for write head + write_weight = self.__get_addressing_weight__(write_key) + # prepare add_vector and erase_vector + erase_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + add_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + erase_vector_expand = paddle.layer.expand( + input=erase_vector, expand_as=self.external_memory) + add_vector_expand = paddle.layer.expand( + input=add_vector, expand_as=self.external_memory) + # prepare scaled add part and erase part + scaled_erase_vector_expand = paddle.layer.scaling( + weight=write_weight, input=erase_vector_expand) + erase_memory_part = paddle.layer.mixed( + input=paddle.layer.dotmul_operator( + a=self.external_memory, + b=scaled_erase_vector_expand, + scale=-1.0)) + add_memory_part = paddle.layer.scaling( + weight=write_weight, input=add_vector_expand) + # update external memory + self.updated_external_memory = paddle.layer.addto( + input=[self.external_memory, add_memory_part, erase_memory_part], + name=self.name) + + def read(self, read_key): + """ + Read head for external memory. + + :param write_key: Key vector for read head to generate addressing + signals. + :type write_key: LayerOutput + :return: Content read from external memory. + :rtype: LayerOutput + """ + # get addressing weight for write head + read_weight = self.__get_addressing_weight__(read_key) + # read content from external memory + scaled = paddle.layer.scaling( + weight=read_weight, input=self.updated_external_memory) + return paddle.layer.pooling( + input=scaled, pooling_type=paddle.pooling.Sum()) + + +def bidirectional_gru_encoder(input, size, word_vec_dim): + """ + Bidirectional GRU encoder. + """ + # token embedding + embeddings = paddle.layer.embedding( + input=input, + size=word_vec_dim, + param_attr=paddle.attr.ParamAttr(name='_encoder_word_embedding')) + # token-level forward and backard encoding for attentions + forward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=False) + backward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=True) + merged = paddle.layer.concat(input=[forward, backward]) + # sequence-level encoding + backward_first = paddle.layer.first_seq(input=backward) + return merged, backward_first + + +def memory_enhanced_decoder(input, target, initial_state, source_context, size, + word_vec_dim, dict_size, is_generating, beam_size): + """ + Memory enhanced GRU decoder. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. vanilla attention mechanism. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both included in this enhanced seq2seq model. + + Here, the bounded memory takes the place of the "state" vector in RNNs. The + state vector in RNNs is a very successfull design enriching the model with + capability to "remember" things in the long run (across multiple sequence + steps). However, such a vector state is somewhat limited to very small + memory bandwith. A bounded memory introduced here could easily increase the + memory capacity under linear complexity cost (rather than quadratic + with vector state). Besides, attention mechasim (with unbounded memory) also + serves as a exteranl memory bank encoding source input information. + + Notice that we take the attention mechanism as a special form of external + memory, with readonly memory bank initialized with encoder states, and a + content-based addressing read head responsable for generating attentional + context. From this view point, we could have a better understanding of + attention mechanism and other types of external memory, and it also enable a + concise and unified implementation for them. + + For more techinical details about external memory, please refer to + `Neural Turing Machines `_. + + For more techinical details about this memory-enhanced decoder, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. This implementation is highly + correlated to this paper with minor differences. + + Also, we reversed the read-write order, for skipping the potential problems + in PaddlePaddle V2 APIs. + See `issue `_. + """ + # prepare initial bounded and unbounded memory + bounded_memory_slot_init = paddle.layer.fc( + input=paddle.layer.pooling( + input=source_context, pooling_type=paddle.pooling.Avg()), + size=size, + act=paddle.activation.Sigmoid()) + bounded_memory_init = paddle.layer.expand( + input=bounded_memory_slot_init, + expand_as=paddle.layer.data( + name='bounded_memory_template', + type=paddle.data_type.integer_value_sequence(0))) + unbounded_memory_init = source_context + + # prepare step function for reccurent group + def recurrent_decoder_step(cur_embedding): + # create hidden state, bounded and unbounded memory. + state = paddle.layer.memory( + name="gru_decoder", size=size, boot_layer=initial_state) + bounded_memory = ExternalMemory( + name="bounded_memory", + mem_slot_size=size, + boot_layer=bounded_memory_init, + readonly=False) + unbounded_memory = ExternalMemory( + name="unbounded_memory", + mem_slot_size=size * 2, + boot_layer=unbounded_memory_init, + readonly=True) + # write bounded memory + bounded_memory.write(state) + # read bounded memory + bounded_memory_read = bounded_memory.read(state) + # prepare key for unbounded memory + key_for_unbounded_memory = paddle.layer.fc( + input=[bounded_memory_read, cur_embedding], + size=size, + act=paddle.activation.Tanh(), + bias_attr=False) + # read unbounded memory (i.e. attention mechanism) + context = unbounded_memory.read(key_for_unbounded_memory) + # gated recurrent unit + gru_inputs = paddle.layer.fc( + input=[context, cur_embedding, bounded_memory_read], + size=size * 3, + act=paddle.activation.Linear(), + bias_attr=False) + gru_output = paddle.layer.gru_step( + name="gru_decoder", input=gru_inputs, output_mem=state, size=size) + # step output + return paddle.layer.fc( + input=[gru_output, context, cur_embedding], + size=dict_size, + act=paddle.activation.Softmax(), + bias_attr=True) + + if not is_generating: + target_embeddings = paddle.layer.embedding( + input=input, + size=word_vec_dim, + param_attr=paddle.attr.ParamAttr(name="_decoder_word_embedding")) + decoder_result = paddle.layer.recurrent_group( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings]) + cost = paddle.layer.classification_cost( + input=decoder_result, label=target) + return cost + else: + target_embeddings = paddle.layer.GeneratedInputV2( + size=dict_size, + embedding_name="_decoder_word_embedding", + embedding_size=word_vec_dim) + beam_gen = paddle.layer.beam_search( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings], + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=100) + return beam_gen + + +def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, + hidden_size, word_vec_dim, dict_size, is_generating, + beam_size): + """ + Seq2Seq Model enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. vanilla attention mechanism. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both included in this enhanced seq2seq model. + + Here, the bounded memory takes the place of the "state" vector in RNNs. The + state vector in RNNs is a very successfull design enriching the model with + capability to "remember" things in the long run (across multiple sequence + steps). However, such a vector state is somewhat limited to very small + memory bandwith. A bounded memory introduced here could easily increase the + memory capacity under linear complexity cost (rather than quadratic + with vector state). Besides, attention mechasim (with unbounded memory) also + serves as a exteranl memory bank encoding source input information. + + Notice that we take the attention mechanism as a special form of external + memory, with readonly memory bank initialized with encoder states, and a + content-based addressing read head responsable for generating attentional + context. From this view point, we could have a better understanding of + attention mechanism and other types of external memory, and it also enable a + concise and unified implementation for them. + + For more techinical details about external memory, please refer to + `Neural Turing Machines `_. + + For more techinical details about this memory-enhanced decoder, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. This implementation is highly + correlated to this paper with minor differences. + + Also, we reversed the read-write order, for skipping the potential problems + in PaddlePaddle V2 APIs. + See `issue `_. + """ + # encoder + context_encodings, sequence_encoding = bidirectional_gru_encoder( + input=encoder_input, size=hidden_size, word_vec_dim=word_vec_dim) + # decoder + return memory_enhanced_decoder( + input=decoder_input, + target=decoder_target, + initial_state=sequence_encoding, + source_context=context_encodings, + size=hidden_size, + word_vec_dim=word_vec_dim, + dict_size=dict_size, + is_generating=is_generating, + beam_size=beam_size) + + +def parse_beam_result(beam_result, dictionary): + """ + Beam result parser. + """ + sentence_list = [] + sentence = [] + for word in beam_result[1]: + if word != -1: + sentence.append(word) + else: + sentence_list.append( + ' '.join([dictionary.get(word) for word in sentence[1:]])) + sentence = [] + beam_probs = beam_result[0] + beam_size = len(beam_probs[0]) + beam_sentences = [ + sentence_list[i:i + beam_size] + for i in range(0, len(sentence_list), beam_size) + ] + return beam_probs, beam_sentences + + +def reader_append_wrapper(reader, append_tuple): + """ + Data reader wrapper for appending extra data to exisiting reader. + """ + + def new_reader(): + for ins in reader(): + yield ins + append_tuple + + return new_reader + + +def train(num_passes): + """ + For training. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(dict_size)) + target_words = paddle.layer.data( + name="target_words", + type=paddle.data_type.integer_value_sequence(dict_size)) + target_next_words = paddle.layer.data( + name='target_next_words', + type=paddle.data_type.integer_value_sequence(dict_size)) + cost = memory_enhanced_seq2seq( + encoder_input=source_words, + decoder_input=target_words, + decoder_target=target_next_words, + hidden_size=hidden_size, + word_vec_dim=word_vec_dim, + dict_size=dict_size, + is_generating=False, + beam_size=beam_size) + + # create parameters and optimizer + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Adam( + learning_rate=5e-5, + gradient_clipping_threshold=5, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + trainer = paddle.trainer.SGD( + cost=cost, parameters=parameters, update_equation=optimizer) + + # create data readers + feeding = { + "source_words": 0, + "target_words": 1, + "target_next_words": 2, + "bounded_memory_template": 3 + } + train_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.train(dict_size), + append_tuple=([0] * memory_slot_num, )) + train_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), + batch_size=batch_size) + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(dict_size), + append_tuple=([0] * memory_slot_num, )) + test_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), + batch_size=batch_size) + + # create event handler + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 10 == 0: + print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( + event.pass_id, event.batch_id, event.cost, event.metrics) + with gzip.open("params.tar.gz", 'w') as f: + parameters.to_tar(f) + else: + sys.stdout.write('.') + sys.stdout.flush() + if isinstance(event, paddle.event.EndPass): + result = trainer.test(reader=test_batch_reader, feeding=feeding) + print "Pass: %d, TestCost: %f, %s" % (event.pass_id, event.cost, + result.metrics) + with gzip.open("params.tar.gz", 'w') as f: + parameters.to_tar(f) + + # run train + trainer.train( + reader=train_batch_reader, + event_handler=event_handler, + num_passes=num_passes, + feeding=feeding) + + +def infer(): + """ + For inferencing. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(dict_size)) + beam_gen = seq2seq( + encoder_input=source_words, + decoder_input=None, + decoder_target=None, + hidden_size=hidden_size, + word_vec_dim=word_vec_dim, + dict_size=dict_size, + is_generating=True, + beam_size=beam_size) + + # load parameters + parameters = paddle.parameters.Parameters.from_tar( + gzip.open("params.tar.gz")) + + # prepare infer data + infer_data = [] + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(dict_size), + append_tuple=([0] * memory_slot_num, )) + for i, item in enumerate(test_append_reader()): + if i < infer_data_num: + infer_data.append((item[0], item[3], )) + + # run inference + beam_result = paddle.infer( + output_layer=beam_gen, + parameters=parameters, + input=infer_data, + field=['prob', 'id']) + + # parse beam result and print + source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) + beam_probs, beam_sentences = parse_beam_result(beam_result, target_dict) + for i in xrange(infer_data_num): + print "\n*******************************************************\n" + print "src:", ' '.join( + [source_dict.get(word) for word in infer_data[i][0]]), "\n" + for j in xrange(beam_size): + print "prob = %f : %s" % (beam_probs[i][j], beam_sentences[i][j]) + + +def main(): + paddle.init(use_gpu=False, trainer_count=1) + train(num_passes=1) + infer() + + +if __name__ == '__main__': + main() From 01839afd77d4c56ba4834f4df5c623279b3bbf12 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Wed, 10 May 2017 12:24:12 +0800 Subject: [PATCH 2/9] Refine and simplify the comment text. --- .../mt_with_external_memory.py | 146 +++++++++--------- 1 file changed, 77 insertions(+), 69 deletions(-) diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py index 1f9bcb0eb8..5734ed9e13 100644 --- a/mt_with_external_memory/mt_with_external_memory.py +++ b/mt_with_external_memory/mt_with_external_memory.py @@ -11,6 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" + This python script is a example model configuration for neural machine + translation with external memory, based on PaddlePaddle V2 APIs. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. vanilla attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories are exploited to enhance the vanilla + Seq2Seq neural machine translation. + + The implementation largely followers the paper + `Memory-enhanced Decoder for Neural Machine Translation + `_, + with some minor differences (will be listed in README.md). + + For details about "external memory", please also refer to + `Neural Turing Machines `_. +""" import paddle.v2 as paddle import sys @@ -27,17 +45,23 @@ class ExternalMemory(object): """ - External neural memory class, with differentiable write/read heads. + External neural memory class. - A simplified Neural Turing Machine (NTM) with only content-based + A simplified Neural Turing Machines (NTM) with only content-based addressing (including content addressing and interpolation, but excluding - convolutional shift and sharpening). It can serve as an external memory - bank, with differential write/read head controllers responsible for storing - and reading information flow dynamically as the model needs. Here, simple - feedforward neural networks are used as the write/read head controllers. + convolutional shift and sharpening). It serves as an external differential + memory bank, with differential write/read head controllers to store + and read information dynamically as needed. Simple feedforward networks are + used as the write/read head controllers. + + The ExternalMemory class could be utilized by many neural network structures + to easily expand their memory bandwidth and accomplish a long-term memory + handling. Besides, some existing mechanism can be realized directly with + the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an + unbounded external memory). - For more techinical details, please refer to the - `NTM paper `_. + For more details, please refer to + `Neural Turing Machines `_. :param name: Memory name. :type name: basestring @@ -72,7 +96,7 @@ def __init__(self, name, mem_slot_size, boot_layer, readonly=False): def __content_addressing__(self, key_vector): """ - Get head's addressing weight via content-based addressing. + Get write/read head's addressing weights via content-based addressing. """ # content-based addressing: a=tanh(W*M + U*key) key_projection = paddle.layer.fc( @@ -126,7 +150,7 @@ def __interpolation__(self, key_vector, addressing_weight): def __get_addressing_weight__(self, key_vector): """ - Get final addressing weight for read/write heads, including content + Get final addressing weights for read/write heads, including content addressing and interpolation. """ # current content-based addressing @@ -138,8 +162,9 @@ def __get_addressing_weight__(self, key_vector): def write(self, write_key): """ Write head for external memory. + It cannot be called if "readonly" set True. - :param write_key: Key vector for write head to generate writing + :param write_key: Key vector for write heads to generate writing content and addressing signals. :type write_key: LayerOutput """ @@ -185,7 +210,7 @@ def read(self, read_key): :param write_key: Key vector for read head to generate addressing signals. :type write_key: LayerOutput - :return: Content read from external memory. + :return: Content (vector) read from external memory. :rtype: LayerOutput """ # get addressing weight for write head @@ -220,41 +245,44 @@ def bidirectional_gru_encoder(input, size, word_vec_dim): def memory_enhanced_decoder(input, target, initial_state, source_context, size, word_vec_dim, dict_size, is_generating, beam_size): """ - Memory enhanced GRU decoder. + GRU sequence decoder enhanced with external memory. The "external memory" refers to two types of memories. - - Unbounded memory: i.e. vanilla attention mechanism. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. - Bounded memory: i.e. external memory in NTM. Both types of external memories can be implemented with - ExternalMemory class, and are both included in this enhanced seq2seq model. - - Here, the bounded memory takes the place of the "state" vector in RNNs. The - state vector in RNNs is a very successfull design enriching the model with - capability to "remember" things in the long run (across multiple sequence - steps). However, such a vector state is somewhat limited to very small - memory bandwith. A bounded memory introduced here could easily increase the - memory capacity under linear complexity cost (rather than quadratic - with vector state). Besides, attention mechasim (with unbounded memory) also - serves as a exteranl memory bank encoding source input information. + ExternalMemory class, and are both exploited in this enhanced RNN decoder. + + The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the + hidden state vector (or cell state in LSTM) carrying information through + a span of sequence time, which is a successful design enriching the model + with capability to "remember" things in the long run. However, such a vector + state is somewhat limited to a very narrow memory bandwidth. External memory + introduced here could easily increase the memory capacity with linear + complexity cost (rather than quadratic for vector state). + + This enhanced decoder expands its "memory passage" through two + ExternalMemory objects: + - Bounded memory for handling long-term information exchange within decoder + itself. A direct expansion of traditional "vector" state. + - Unbounded memory for handling source language's token-wise information. + Exactly the attention mechanism over Seq2Seq. Notice that we take the attention mechanism as a special form of external - memory, with readonly memory bank initialized with encoder states, and a - content-based addressing read head responsable for generating attentional - context. From this view point, we could have a better understanding of - attention mechanism and other types of external memory, and it also enable a - concise and unified implementation for them. + memory, with read-only memory bank initialized with encoder states, and a + read head with content-based addressing (attention). From this view point, + we arrive at a better understanding of attention mechanism itself and other + external memory, and a concise and unified implementation for them. - For more techinical details about external memory, please refer to + For more details about external memory, please refer to `Neural Turing Machines `_. - For more techinical details about this memory-enhanced decoder, please + For more details about this memory-enhanced decoder, please refer to `Memory-enhanced Decoder for Neural Machine Translation `_. This implementation is highly - correlated to this paper with minor differences. - - Also, we reversed the read-write order, for skipping the potential problems - in PaddlePaddle V2 APIs. - See `issue `_. + correlated to this paper, but with minor differences (e.g. put "write" + before "read" to bypass a potential bug in V2 APIs. See + (`issue `_). """ # prepare initial bounded and unbounded memory bounded_memory_slot_init = paddle.layer.fc( @@ -346,38 +374,19 @@ def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, Seq2Seq Model enhanced with external memory. The "external memory" refers to two types of memories. - - Unbounded memory: i.e. vanilla attention mechanism. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. - Bounded memory: i.e. external memory in NTM. Both types of external memories can be implemented with - ExternalMemory class, and are both included in this enhanced seq2seq model. - - Here, the bounded memory takes the place of the "state" vector in RNNs. The - state vector in RNNs is a very successfull design enriching the model with - capability to "remember" things in the long run (across multiple sequence - steps). However, such a vector state is somewhat limited to very small - memory bandwith. A bounded memory introduced here could easily increase the - memory capacity under linear complexity cost (rather than quadratic - with vector state). Besides, attention mechasim (with unbounded memory) also - serves as a exteranl memory bank encoding source input information. - - Notice that we take the attention mechanism as a special form of external - memory, with readonly memory bank initialized with encoder states, and a - content-based addressing read head responsable for generating attentional - context. From this view point, we could have a better understanding of - attention mechanism and other types of external memory, and it also enable a - concise and unified implementation for them. + ExternalMemory class, and are both exploited in this Seq2Seq model. - For more techinical details about external memory, please refer to + Please refer to the function comments of memory_enhanced_decoder(...). + + For more details about external memory, please refer to `Neural Turing Machines `_. - For more techinical details about this memory-enhanced decoder, please + For more details about this memory-enhanced Seq2Seq, please refer to `Memory-enhanced Decoder for Neural Machine Translation - `_. This implementation is highly - correlated to this paper with minor differences. - - Also, we reversed the read-write order, for skipping the potential problems - in PaddlePaddle V2 APIs. - See `issue `_. + `_. """ # encoder context_encodings, sequence_encoding = bidirectional_gru_encoder( @@ -395,9 +404,9 @@ def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, beam_size=beam_size) -def parse_beam_result(beam_result, dictionary): +def parse_beam_search_result(beam_result, dictionary): """ - Beam result parser. + Beam search result parser. """ sentence_list = [] sentence = [] @@ -488,8 +497,6 @@ def event_handler(event): if event.batch_id % 10 == 0: print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( event.pass_id, event.batch_id, event.cost, event.metrics) - with gzip.open("params.tar.gz", 'w') as f: - parameters.to_tar(f) else: sys.stdout.write('.') sys.stdout.flush() @@ -548,9 +555,10 @@ def infer(): # parse beam result and print source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) - beam_probs, beam_sentences = parse_beam_result(beam_result, target_dict) + beam_probs, beam_sentences = parse_beam_search_result(beam_result, + target_dict) for i in xrange(infer_data_num): - print "\n*******************************************************\n" + print "\n***************************************************\n" print "src:", ' '.join( [source_dict.get(word) for word in infer_data[i][0]]), "\n" for j in xrange(beam_size): @@ -558,7 +566,7 @@ def infer(): def main(): - paddle.init(use_gpu=False, trainer_count=1) + paddle.init(use_gpu=False, trainer_count=8) train(num_passes=1) infer() From cdbf36dae9f23cc62b20d749b0e952ae631aa9eb Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Thu, 11 May 2017 15:27:04 +0800 Subject: [PATCH 3/9] Add chapter "Model Overview" to README.txt. --- mt_with_external_memory/README.md | 102 ++++++++++++++++++ .../image/memory_enhanced_decoder.png | Bin 0 -> 96178 bytes .../image/neural_turing_machine_arch.png | Bin 0 -> 34465 bytes .../image/turing_machine_cartoon.gif | Bin 0 -> 13723 bytes 4 files changed, 102 insertions(+) create mode 100644 mt_with_external_memory/image/memory_enhanced_decoder.png create mode 100644 mt_with_external_memory/image/neural_turing_machine_arch.png create mode 100644 mt_with_external_memory/image/turing_machine_cartoon.gif diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index a0990367ef..de398b40b7 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -1 +1,103 @@ +# Neural Machine Translation with External Memory + +带**外部存储或记忆**（External Memory）模块的神经机器翻译模型（Neural Machine Translation, NTM），是神经机器翻译模型的一个重要扩展。它利用可微分（differentiable）的外部记忆模块（其读写控制器以神经网络方式实现），来拓展神经翻译模型内部的工作记忆（Working Memory）的带宽或容量，即作为一个高效的 “外部知识库”，辅助完成翻译等任务中大量信息的临时存储和提取，有效提升模型效果。 + +该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务。例如：机器阅读理解 / 问答（Machine Reading Comprehension / Question Answering)、多轮对话（Multi-turn Dialog）、其他长文本生成任务等。同时，“记忆” 作为认知的重要部分之一，可被用于强化其他多种机器学习模型的表现。该示例仅基于神经机器翻译模型（单指 Seq2Seq， 序列到序列）结合外部记忆机制，起到抛砖引玉的作用，并解释 PaddlePaddle 在搭建此类模型时的灵活性。 + +本文所采用的外部记忆机制，主要指**神经图灵机**\[[1](#references)\]。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度神经网络的背景下，涌现出一系列有意思的工作。例如：记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。除神经图灵机外，其他均不在本文的讨论范围内。 + +本文的实现主要参考论文\[[2](#references)\]，但略有不同。并基于 PaddlePaddle V2 APIs。初次使用请参考PaddlePaddle [安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。 + +## Model Overview + +### Introduction + +记忆（Memory)，是人类（或动物）认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（不同于感知）成为可能。记忆，同样是机器学习模型需要拥有的关键能力之一。 + +可以说，任何机器学习模型，原生就拥有一定的记忆能力，无论它是参数模型，还是非参模型，无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络参数即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的，在预测时，模型是静态一致的，不拥有额外的跨时间步的记忆能力。 + +#### 动态记忆 #1 --- RNNs 中的隐状态向量 + +当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个相对稳定的信息通路。带有隐状态向量 $h$（Hidden State 或 Cell State $c$）的 RNN 模型 ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可动态地往上叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 + +注意，尽管 $c$ 或者叠加 Gate 结构的 $h$ 的存在，从优化和概率论的角度看通常都有着不同的解释（优化角度：为了在梯度计算中改良 Jacobian 矩阵的特征值，以减轻长程梯度衰减问题，降低优化难度；概率论角度：使得序列具有马尔科夫性），但不妨碍我们从直觉的角度将它理解为某种 “线性” 的且较 “直接” 的 “记忆通道”。 + +#### 动态记忆 #2 --- Seq2Seq 中的注意力机制 +然而这样的一个向量化的 $h$ 的信息带宽极为有限。在 Seq2Seq 序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（encoder）转移至解码器（decoder）的过程中的信息丢失，即通常所说的 “依赖一个状态向量来编码整个源句子，有着严重的信息损失”。 + +于是，注意力机制（Attention Mechanism）被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的状态向量，而是依赖一个向量组（其中的每个向量记录编码器处理每个token时的状态向量），并通过软性的（soft）、广泛分布的（distributed) 的注意强度（attention strength) 来分配注意力资源，提取（线性加权）信息用于序列的不同位置的符号生成。 + +这里的 “向量组” 蕴含着更大更精准的信息，它可以被认为是一个无界的外部存储器（Unbounded External Memory）。在源语言的编码（encoding）完成时，该外部存储即被初始化为各 token 的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于 NTM的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址 (Location-based Addressing)。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，这有待于进一步的探索。另外，这里的 “无界” 指的是 “记忆向量组” 的向量个数非固定，而是随着源语言的 token 数的变化而变化，数量不受限。 + +#### 动态记忆 #3 --- 神经图灵机 + +图灵机（Turing Machines)，或冯诺依曼体系（Von Neumann Architecture），计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#references)\]试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。 + +
+
+图1. 图灵机（漫画）。 +
+ +图灵机的存储机制，被形象比喻成一个纸带（tape），在这个纸带上有读写头（write/read heads）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （contoller) 控制（见图1）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”（$n$为记忆槽/记忆向量的数量，$m$为记忆向量的长度），以前馈神经网络（MLP）或者 循环神经网络（RNN）来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（不赘述，请参考论文\[[1](#references)\]），并最终写入或读出信息，供其他网络使用（见图2）。 + +
+
+图2. 神经图灵机结构示意图。 +
+ +和上述的注意力机制相比，神经图灵机有着很多相同点和不同点。相同在于：均利用外部存储和其上的相关操作，矩阵（或向量组）形式的存储，可微分的寻址方式。不同在于：神经图灵机有读有写（真正意义上的存储器），并且其寻址不仅限于基于内容的寻址，同时结合基于位置的寻址（使得例如 “长序列复制” 等需要 “连续寻址” 的任务更容易），此外它是有界的（bounded)。 + +#### 三种记忆混合，强化神经机器翻译模型 + +尽管在一般的 Seq2Seq 模型中，注意力机制都已经是标配。然而，注意机制的外部存储仅仅是用于存储源语言的信息。在解码器内部，信息通路仍然是依赖于 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充（或替换）解码器内部的单向量信息通路，成为自然而然的想法。 + +当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储（状态-状态转移矩阵）和计算复杂度为代价。而基于神经图灵机的记忆扩展的代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的，而 NTM 的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净。 + +所以，在该实现中，RNNs 原有的状态向量 $h$ 或 $c$、 Seq2Seq 常见的注意力机制，被保留；同时，类似 NTM （简化版，无基于位置的寻址） 的有界外部记忆网络被以补充 $h$ 的形式加入。整体的模型实现则类似于论文\[[2](#references)\]，但有少量差异。同时参考该模型的另外一份基于V1 APIs [配置](https://github.com/lcy-seso/paddle_confs_v1/blob/master/mt_with_external_memory/gru_attention_with_external_memory.conf)， 同样有少量差异。具体讨论于 [Further Discussion](#discussions) 一章。 + +注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 ExternalMemory 类。只是前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。 + +### Architecture +网络总体结构基于传统的 Seq2Seq 结构，即RNNsearch\[[3](#references)\] 结构基础上叠加简化版 NTM\[[1](#references)\]。 + +- 编码器（encoder）采用标准**双向GRU结构**（非 stack），不再赘述。 +- 解码器（decoder）采用和论文\[[2](#references)\] 基本相同的结构，见图3（修改自论文\[[2](#references)\]) 。 + +
+
+图3. 通过外部记忆增强的解码器结构示意图。 +
+ +解码器结构图，解释如下： + +1. $M_{t-1}^B$ 和 $M_t^B$ 为有界外部存储矩阵，前者为上一时间步存储矩阵的状态，后者为当前时间步的状态。$\textrm{read}^B$ 和 $\textrm{write}$ 为对应的读写头（包含其控制器）。$r_t$ 为对应的读出向量。 +2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头（无 写头），二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量（即 attention context)。 +3. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#references)\] 基本一致。 + +## Implementation + TBD + +## Getting Started + +### Prepare the Training Data + +TBD + +### Training a Model + +TBD + +### Generating Sequences + +TBD + +## Discussions + +TBD + +## References + +1. Alex Graves, Greg Wayne, Ivo Danihelka, [Neural Turing Machines](https://arxiv.org/abs/1410.5401). arXiv preprint arXiv:1410.5401, 2014. +2. Mingxuan Wang, Zhengdong Lu, Hang Li, Qun Liu，[Memory-enhanced Decoder Neural Machine Translation](https://arxiv.org/abs/1606.02003). arXiv preprint arXiv:1606.02003, 2016. +3. Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio, [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473). arXiv preprint arXiv:1409.0473, 2014. \ No newline at end of file diff --git a/mt_with_external_memory/image/memory_enhanced_decoder.png b/mt_with_external_memory/image/memory_enhanced_decoder.png new file mode 100644 index 0000000000000000000000000000000000000000..e80cecaf49932b60cd9c623f825d8caff9e8c182 GIT binary patch literal 96178 zcmeFZbySt@);muLBY@?!h?6H z3CT2}pq}Dch>9w{5fvph$#%Y7!DV+@K1hl=zC}rCCmm2nDc8PF>x(?6b{wJZ+^uk-UnVw^H+`= zX>1QEp8F>7P~wE@dU_oButI1pLKwtr1G2KxX0aYn&{;wxG|=35+R2;PxVUIp;pe_d z7d226`z6{T#&FLxoP<}t*&*0fmjyv5jjMSiC zAVOi{uR77Hx0wO56zVqC%zg8zKFIR>U|l!z&*%B!Xj+Boqkh336lW%-DU3 zhwvK#LwjRwO{7pNaQgLG5Z9J(SPBnbT$eOy$1j~1BIlb(C*@id4C`l7)fpPeeyUB$e2$4!hTMtoP88DgY-s=kuBUGoVOm76*31ujd0?aT zwu5ZaKbgEg^d!cRH&N&}%HAlVp0HHm(_CbSuck+%B0er+we-WsPi=Z>7W;AGJHuzc z9l?l_1ttWNF!Tq@MIVce;2r;_BvbdR6-~y)etF`p8oz?iVo8Z19>z^5a?2=(@r7tGnT(RiX)_CQ7oau!L3VIKIppGIoa~w_WW$v@+VWpn<43`wj!=D~SQGu#@ty zCg;=YHZF>lPU&T$?d{1*OT{pn2EUwJNzBVdzM42TjnD;$COC8`VW8dm2^zp)TKyoy zH-XW+5$b@tr6NXxr6+^h@BZ%}eFWU^71|H5#Cd%>> zYMVoKdNw%ixD7<_&LFNHyXyL&KowIdrrhiHutF_gszCiYXxVwX*y^DFq%{PsT1t^- zBBaLnGE*2j^swM36i+Kua=X{AAGRIBA{kVGzZ3>aY^zi4Zvrv|hYX|*fAOqm@&4=v zPx4^rTIEaeiJ*dnpS*+5ZF9GSbMi}F7AOo8IK+DDe_`-U7z0a0^iME+cGxp=ykO$r z{%#b|Qe0ROFUV=7c-ke!dUZ^Yltbgm6ME6U!Cw(`N2WdYY4+ zpT}VtE-$33Nc#F|gm~_kt#<-%0~p0MzTm!NuO}^%RQRg#GxZ(kH5PF@)XOLqX8u8B z$F|#2DS85#4%lh0LRh+<{IZXl*j^)3Wj0n&Hv!o8Jj0zu2G;Pa_FRM`J)5g^N77zQ zdQq8OC9C|GWcLw5lcjoT#_SQ;g69r!H^3cK5USs zFvHr!(7>v}*a>Fnls96s4;G`ah^Ch!`6h)GrYiL6r8UJKbvlJ}%(WD0-?52qIWc#1 zn(Rg#e%wIcO)qty?wa|UAvYB_B{yfC6nEsjRo#R`!`kPy;=Z#!RF!zzmpJIs^i*-> z!OFtiQrwd6Lhf`-OeIVQuLQL+s9O}9Med_-#ji-N6jX8ZxhJA{5?*B&XYYThRyll%aPVkUX~*fDXd9K|RbDHfDy0=`e$y;= z6Fd6++b=dPwk$RiOU9yEIpLRm&YZMpzmuu_HO;M%KuGANEL(4pYvqk>l3v^d&7uk?OlW;*>)|cXy7aethRe=}bYvDTKtl=Ebb(yaXtc=nO z8V76JF6%<--vqyjc~jK)MCJ33B#UN?sx4Hu?36IA#2LH_G{^DzQNE7G{@c%RmMbQ0 z71Kte7H>+Os@AmCwHK?i-(G3rzMXh`rpZ>=QG~C}soC<`fQ#I{eHID-rB{W0WncA- zspFD*7st3unRVPAA+IBEwQj-!s^@3#tUH(+iJK`jJT#r4A~aW`t;jf{B<^Y+A7=*_ zEG}ZFZC4H_dHcm(uQ~Avs(#a)0dWogf|4DWDYGdB8+jWzo1lezhZ=_who#MXuk1dt z{Z3tMPySPWO%h(ddj6CQ+Kf5=hNja-O}z=ds3z1#?!|_t%&X)plk43J_x1Oybcg&K zH*F(Bwi)_if*Yi3`oEV*vam4l6$uV$6a|_2OF8VAds*JeE6OmFbJ4q5AMRo7VYo=i z^%~?k{%QPKVK`ozT-smSv$C<`fB5~)pKm|k?7ZX;W)5AJd@bhR8{O+1gZ@=E`}pg9 zvTPa=d!6O+*t@wub))!3e^}Uz&bu0;&LVV)Jq3qf%i~NvFL@@&-u&`j)ZbS`OT_he zK&N|%M(pc2w>J@e-_|bI7Wzb0V7^-=|Kgmmw>%ox9+Uk(^x4Ite&cj^i`tB4*kFh& zRz&KJ)J)RxC)~b#Gt`{GZ#MG8CO2iA<@yed+&kQNI_Ne87h0bs-)haCI7OX0Z`~2- z5v&tD;SKCe}m?vCf)<-Txb zb47TJ0B7jGAR@*kH?5h49K~%-p)$&pogp&Pp5GqDwrWn2lEW7!SYTa!wToMki3^8! z$AOU9kuk=jVY{DQ$A99rv6mb*sF1!?-)hkI3(gzmBi_EQm{xq%pKYVmF4M3jhBunO zDpftV+H;y(VDFT*3l-^Unwti_XR6mP9TtzzanI>EiZ0dLY3p<>Ro?1dn^JkMuj^a; z7F{}L>26vHoU- zW4&99Sn_V(q}F2SJ5vHd0%w6&!8XTxYrV3o4!q&S;-W=ubepk7_4>UNm&J2lUVoQy z4ML5igBSbs^H)w9x5X$eb+_sj+K#SU`z4;2p7j?J2mIQrIw}bg1V~kMwrT^~~8vR07n6 zh&K`GJ)%)H9Ce(;Hyzhe{6ii?YSiQMg#vH5>rU5te`KqOGPpB5$sJR;s2+aT|D-jk}e~sj(xrI(#EvQZvN_o@t*wN*;HBy zyGyOhYiIdgzM1t2=j-~1vlC7c8=X1=`-&6w`6Aw$+{OLoj=GauoAZUK3*-aTCH+gk z!>qM0zdDX3-$vy>7@kMf+vM(J^WXDZM;VhW9hhD;dr^2D=h$wk%#IzK*N<2NOSIib z-#u_+Sr=a#XiVyO(&hW=q3$YStKh4z`@;lBGS~FQ+e44Fq5hiZwik_#Zc{us?5BUc zhi_KUsYuxc_V3n?x7s%YMn(e7eaO9kTw0!2ZD~)}AW(8sQo7Sa%Q)b)8iXd^V*hHU=H$eu@|ypx1k z=jX<9FWX;x@>fcG1RZPjS&?LAD4i79Cyn`wZmT$4HQG9qt3xpV2` z-pp@fJ*>FJE^>UI`SW$yC`c5L?4-0Dp`h@nApb$XQGRg<1qGdMp{nVmDJRQgY-`PE zXku$*%IIco2cCw4;&bBxFRe|T49VQAt!x~5-1sRTpWp$nA#XELkUc))^q!wWQ%;dg z)Yie2jGgf{BQu2nG8q{epM!}RkFuD=--m;L@l(8Wa0Mr!@fNOpFPe;xTRpZt9!9~0yN|8k(eN$c@lAT9x9KBj+)UI2Oher5pN z29brBf-3kM268pv;_$%#X#Vy$pVC@3MQH)6u7ZqU2w2ww5x*FC-*7ugfKXRHd%)_|2YCYtH^vpLI#cXmmf|r*@Dmt?O#8^ z$}-sC7lL{5mmeEMMdpSN?J9r$_5pFzdRp!hEzBAYEf`7aw@4upcMAf(|F|1amn8Ei!O zFB@(}g+?~xRCEjdFV_Yx2L1oPn7@es|Nh0OXXY0biF_lXxsQ&HN=Zrid94fo z`*r=4&Z?^;U}tB4XK884Pe7j7vBK;mVr6CZvohI#Ca^eJz?cZ&H>efx=i`P2qrw2=Z zaQ-fC*x%-JOy1ON@!=0dBbII>+PE9s_*av1lWe6*)+l=WY4oe4r-l@w9#wAkfFuWP zvS{gvR}m|mBb_kvhB{v5zZrrMH86Bcv%3t$o7ZrT@9|ZbYs$0!-Gq36ks$xS@d=qW zKC5YFdhP0`#yyeN&OiUdnfT!hQc6a4_9xI$T9TUOj#=z>+zzsh80^p22|H4AiFKt3 ztE4g)Ap)`VI0V!sz%l%rGW|rd3rL|HyynL|w?N*QWc+SmV=5f>Z=S>nIB}f+uRf9J z?>=2>s99}A0PJCI`#+TG1r%+vWF%2Y0-c7@>7P+d(atR=3Lc*4BYY=MH-^&*U+=jD zhjM@X&SCf#6G%659np~vwzet*1wv${VZB+~xv ziACg%@84fq%~hjTRaO02XyB}O-kBt)OQ|U>#Yjj<_%r$y!DP^Yk0$wJnT96yzufc( z1M=lCgW~-B_ThBii0`L6Q|M)7WyBmZmf`;XFda(;8cZ)D4-k61?Zy7)%*6Z1A~UL6A*mZtL}YxVu1@X!Nc6g){gH zb@ty*_cPJ2#$O99>+UGZx05o8&p$1VXl;Gn@XyaL)+kGvvA<74s54qWq6-8IcwZab z-(ErM>j(SKSZj!Pzp;gLw6`y_k2_dsn8Ij1p2Kqu+6in~Pt<;T5lbawH?AaEH}L$> z5oZwk07m^^gFIOiQtdX`R5Kn)}Xad;Eu#oSZfPn=n}a z06!*wtzkv}i+RT~?;97s)1RWXTi;XXtG90sT6*GXRl>LCT<7f(_UkrDjCm^O207B7 zs<+YS9v&aEeHa{+%}su}uuk*44rcRgN}oxC#Wi#@Dc#i6)Mh}l#wIcT;67e?f`Jb~rVm;96C?TWKu6XsT^6ck_(H z8WMqA%&iqq@~ibE*)lVx<|?(GPUOy}S1A;Q`J``G^mBxL0fTHm>f!U3?AK5{j;PkN zUC5mr7|h)`>3aOUp@JjNT}`_rH$`D&$9k4Na9d?>cs6%ZyhZ*@%QUs?FsOv?teT=( zFAx&z^vneKOKL1Cp5-f4nbNZt&3f=Ro^7r7o@NPuOcP;-?U1CL`~W+Psaatbam20p zG_dN#*+qmeZkO-4moj3fORA!xxVU(MDk366{-`!LhMJtb?P@hjuzCPH7+blc_aZl0 zjWGC-xO=TyIa}k_+l{xK-o}jY_OPS&X1^gOE$!*iMjJE&(a-Po$6BA}Y#N5#y03nU zqO*TjJKdYBk;^cUu9ZlR61aZ1wz1)&$$lg`>P9lB3$NhqApmx}Z{+4VT&M%2#F_{> z4#|HWC;e*NV|(g{K!Mfc5uc0(enP<)xsCbgrH@X;RVsIKR{ujD4EG=$f6sHW<=!N> zbEY()!EcQm#%L z7rze~o6OLxrk-lRWV{WVuBq69&Y2src^1wKp zir%(4Flv^P{n_qDhD3TH8f**{B7zqEPX=h~W~KXvvXG4(nMSMd{j!)}UG0f-?Z-5~ z!_a?D;B&<-%C}kA@FWp!64Ok+uX-i%P@Q=oKV%al?HJ$F%d1hv>t^hG_3`&umC~GK zypi8r4+jtTBecya7IDmO`XR9ytv*sJ5_%Drt| zEaG#|!m+g#xx0S1HET>~W39DjTDg|;cL1~##xq^`OdrC1#g-Z2ML|JH2ar@Cp$pS2^YG|5B1J3a2I96AM*`TE zIt-CvH7@5zcj4A4%#8>_PM)!`G1R-m)+gv(>lFIO(I|ck02mHWPiy9;6%)&T9Vqu9 zGEBswI6V|+9x#s`;XPBCku+IKF#>qZrHSd-cu5qz+!JTLLR zE3lVu##|e+b?N;vr=jm%b+T8pwErtnWnyAt7(myb(;nnJJhQ>M3OmeUJHwJ~yOGEv zF+!dI?Chl+K7|BeO<|C}A&1g;WWsvk4myZwAshd|HXF&6=0e>}isHbZ}=#H=04R*8H9gp=rV+($jUztF(zd9lBD3jlapDu*@e zxoINGhXnwnV+egwc`~o1U4uM2^ZW zLNA$=ljjm4b}(BtK-*7z*-pVd+-nHpp?GXD+O7UI64Ut;wQ2TVXcj&@aiz%d>X<#z7)aM^O{m><7|X zl+H-+*oT_P{{3CAQe?BB!dkzcjwsmj@^ZVsCl1H2LNS`S_G?$kz8i~%vx@RfS;e0K zsJnc7G7_W-U7}lO^a}p|c2iKauoR_CgRQgU;TgLnPNx)mASxj|dE;n>`H0I@oqRAM z-fubRG8#Yo?Jq$j?dh(INaV@n87VHNJI6cMXFDuTe-wk&%E@7UlS($>USQC|cHVDe zRJ;klIIOf7qsW=DqK0V4WxCK(vp4yq|IrmdUU6sC;9-fR_ywuo?HYy>2G=@Pc$ z3>%Z@9ja9yeD>*?-$%Db?2?xI`lDA8sdEW*cyJx!_`0xV&z^BuO+&Ht27U#ccwUllWf)+*(Ls3;W(>@9q61kP8uso z`aawlZoiR&EpUrfto6DOS=R{H=WJU(Ju~c1W-%rcbfTuFCd~E(Xe4l9gCY(Vt^$du z9^#E?lJB4r#AAV9_9FvH)EwC~k|ud_*SS*nrmu$fx=>0o3o=XlJS*A8ah=|J*8J_& z3Hf&`QNI2V$6;;+BTwxPhN|AO9+I%UJQ@v`0sW6^%#PYWhBx3fmoJ>9s|(u)Q`oe> zt34%P?bK3KTvi(DtP$;Yp3-$qSD>?Sl&+SnO$mXer);I2N&I3W3;~Zn1EF$FYTDYw zFtAFEq^rGP$EpsKFzSX2itGDnNvC#qqfr9+&!>$8eWJCeXWe0U;;oR)2=`(1fu~*6 zl-=a8ng8$v;dyGz0AYWesK7YWg`wwO)s&aVKCm6^*+Eg5Nz5#R>PRY?$5bi-5H8Xm zU6`>-MVJTp6T6%&MFoQ;c_TKX`7H|O3$z%|c@0{jjK3Hv6qmhH0ICVXkQb})4UBhs z>>a>iqV!MD%MjRXny<`O-T!oUcAkHjPs5y35SGvN*-Up0Hz(K8T5}vK&?pJ?QlD|| zW$lvokODC@-O_zMeisktwNxt}Oj@ipiWxjPG*++>cuo8voy1748XP{&=MG#kB`@!s z+e zBI=N{Mp)3g^g_fK1<)&c>qi-vE`KQ)S zh8?urxSK@y*VvvKeUdU}bEqS)yNx;G;h{<9rRX?$u}lhl@%>7_GT6o=D<@Qn4WhqH z+trMDCigmJnuAOpp-pYuG+I90Ucyw^sL>`0Q=@?u!(769?;)ACUAi`m=0(Cb`U}a1 zvq|j;LiNQ;BTMHbq+sj3v|~}=$(9z@-=Cq-Bs){c#`*aIO7bR&NroC+f(}?YX71w4 zF9(o>`1mxRfWvZ}esnu+rL9ibbBJ4VTSd(sXLKNb(SVL@))0cA)L4vc=3Wte7ne>z zK=24CbXMwt^BQ2l@+2li-yN_ZXePlKY{ptUd-0PUOs~7zH4oADWm-T?D7<0O&q8#g zOkP*r9bjx2{E247Pa!JP+r!)7!FT=V>#)*S3GS&9{iO*R?_ttH|CBHvt)l=BYEviY z(UQ z)q#|5k2#7L0j`4g6`v%`9?+c{^L94Kg&M$t3#Fqef9i_`v{hS)n%jUn!@K)p$=5dg zRi<4#5(#Rx7G^30S&+Y#0Ps-kB?%34Z(q!c5UDbASW#iA)eNhAADWXnvyUQnw7(Yp z;)ljYKI6A)iW-Rtlz_|pfI$Iln}bNU0>4)g&yz2``z z#w?(_ue2hW=Nt>*6mS6F(YzeL3!&;-!+}Rn9sLpoK`o8cz}GKUtR|deAO_OToL5#_ z&UQUm#2oYCrt$>z zgxk3m@<#BMlZI$XBZcn)WWPPN_fP;<}k_7WT`Y?!r6N1ex zT^?CJU{5!D^A^`_JjcPuzY$DMMFePky382C<81p=XQvn^_pR$IMn*KwB`>z`^6A?x zWV-YzbJEV3&nn^|2O|cP?qv-=0|JQw{*W8XGu?V0`9pR#1%$G#1uhOCm!HKt&lBBg zlRL__XK27_6ah0zw9Vgp4W@vlgGSyk*dv&vdxqHC6usoWNvz{CW17tG$&ul6y=9Q6 zF0=!lR=j;G50UylxYmZv_6mqnFN2|G`bfvub}tq6^dbSML6u3QXPGpJMLGYTlk>?j zj@OzH&M`MQ`2*yRcfqPwf{qd|fG>Rt@uj09RynS)B!HZlcbT*9?gGU5We4fKU#d+# zl9H0rBJMMI4JGCvVd`=uu-u9yINt4bJOc>Oh-hGZ+bDXW_#x(WgDX>a6e)V0bPXR7 z8iGW`^H0Dso<3tHyL@G(5;V=K zDURLaS=;9TMk!d_a_`ZLxhVxmGs?2588aZXEQrufbvuy(EDE3jLc`eOQlBesLmyZIh?{5xx=}mU0%PtRmAHv4R$I(s#3O@RC>v$&aL1bnD)+b-^ zF60B_F+i4p_IucH!it*gLbrJ{!!vxMH1+dmo^VoZ?2l7=o~SL?6D2%@*u`kb8+LnI z)qwpa`v&el$myc&7zj`p8;h)bIeAah`o17yu1>C_ypM&5eRtcrl@p+ zaE_YJ)P$liFEALv_B(Q@XV8Ew4It)Z#ovAn8|tSsFB8c)pjOJy4QR{2+zb!oBprv4 zvh{fCsQ^(Du=*eIYM?)4yP%*Kq?=R@frG*#V09@T{|Q8{4yh831)ctuQN!srDbMD0rikgD< zR!9h`H=*nR6h!&1*5dL?N-pvk9m0M9zOC_UBQ?-2aZ8Hpe^S+GO5iJ_cUaYv!A8Xp zL1r(;6FPB92+fUq0|0jyfV)R0BZ8hplE^0Rdw_wn31`jC$#M6xrZN8p2#-bM1%PMT z6G4tN?ZQAP0mNkhZVP%JDiECux{meSJkvYyV$l=*7b62h7swhTJo;ib+22oQ<>evX zU7vLTC3w5NgEe@pwGl+>^Xwx$s^!tA3gI@Q;iiFVaL539ps>+`xR8Q6>jm_HikGQ=4)bHm9bF?K;%Zf)D9 ziC72GIt(DXeg4UGAUZ3gf73{-xxT>1aJ2yVA4VFQ*7i>*Ebhl94IOMGNU-c5%F9`a zyiZ2~uS>i!lo|v;A${J?d&%{GhJY78#X+Fnr(iY%fRzjy9n^7VL=^||V!(VPoHl9X z$@(5!R5~78IhcLI)~_Ks4hQ5SV3ELml5HWXe*~NQr=j72v)G`={XZ);GPsi%Ns^GW zovGCc4gIb{tqOWt&UlnB{|76vgrTCMdM4n-g^YqyX3z$+KThru=FgpHtECQT5i(_< z36hp06${|L?_YwNy=^*VlVbo2qM0np z|EihF%$u8=luS%fBj|9AoSG0tv;(NkQCRo`%YqYdJuBm{tEoB>UvqMHoUJ3op_{j zgsuEg7=cuN)p{;3-A=d05SlzM9#e!jaW4(mdSj$yWt~iqZs_1DmN40X!h-O(6b#gW8pj7x`Hs% z&Q4d~+%mWwU)nStNvy4{{h7Iq{cdoMqK+pGvCn1@E6*cHOI||c6$zq%t61RXog9S5 zW@h(f^b0*yd#q`4iE;1C-{ts08F(04jxc*gERWMs4&Ztl3wR`0-5 zO(f?`3`luhpQ(bBk%XQpjf!U(*QZ{Px119;Kc}Yt9G?zoYH(fAFS#cmZMi@npHN+i zm0@gCQ2_mUTQfL=#f!^ykA{i~G1Tp+>Ysp87eXSUnron}vCH1qhgDjBBdI9ANIE zeg2Q;&I2*`GLttDbI$_KPu=3@d3-CVqDM@mMDc9X=r~WPL3~VXTpWKB5eT$g>V=vg z!b+M@H}zefthu}?7RCLT&u|C1*r|HGA+KV-=H|~`hkq-w6-}vK6zMl%X{5WdDtGlB z14`cS{_bWcNi*@gLG@EAl*cHy11?p&Czk3l4uHs{l_NZ-0;9~F>A7Dw-RNFXmOi@H z7YF+ILFqZHwzgJ2Y-_576hN@RI#+>k3pOU}lfj}_@M_w4`QO#F(ck^MU2B(o-JhdU z9~OA{Hfk5zH4bR8VBwGOTmPFqFt#;XjJ;gNs>`mzX&d`4*MX~{^8vyJWbFKjo+Iu1 z)UtLmpfb`_E64G86W_p1$U(mqSTI+cox9w2&S%)k0b44m*pKtS(x481U9QBuQ~5V7 zLxgAum|FVo1#C^{u2gl(AXNz2ucEwe$ZAIDgcE^4m9 zVvSlNqt`^GWkYS9o--+2Mc7k)2r)z^0`d3ndMakXRr&znzU$g)NYI*YFiH3dKO?q) zV_>2bbgYb;<#WKX!BzOQFv~$~{KfJUpn9asN6HY@VdDD8S{AEGjG z76g*_e+1YgQxt|aLl_2P_u|GrO*p7&Cw>KDj56GCTu;bm=H#p)R!M5906qoOOV<}0 z9Gp$uhnIqMc6R^_lhD#ik^ExfE3@KtK4VTe!WH$L9XTf_Cy~FS@oLlAZF@0QNkgNe zMBum@_nF7qY4;aNk|5DYK0E{>u5p8}CqIgQv?16w`9Iv>q`O$Zk$=-gusH%6`a5FA z8d~hp>sWt`W&2_ks0yClr0OspF&{Ty9(UFv1$yzo_ppXJ05qHy20cEY7-AjXp3kDj@;xJ|I#v$`!bn z3sgm3JLiueYezOhngtAuC$d44y;iW`yh9E-*BER0TZ!}3&(D?WU2A?0qsbI zMWK@qYfT}Nwk_~TA#WrI7=`KCCRH`Q6q=xydu;sv_QTxxm>d6#YYTKRp8j&4X>TKN z4>=0_PC#9ygC60R7#Z-Zh`UZ4DFAce!7#QxHtSrUZuvyf!hIvsiiepj77f)P0CnwQ z4q;m3zGUTZbIxmW93(v9U1=062pg z{Dh#r06jlHpPtYVFc|knI9Im=C6aF?zK>2hoVuOWEG<`{FNv>I>)ZWNTD3oUqCi=r z@Q`?&?|(Hfv+?^6Hg)x7zx2iLlPElciC;{g4i`UiPc41W*LBh5EdzN+Q1IyP7QTK+ zyv+{@FD=Cw%{Gs&q=$~#gT!pR8@zCV-_Hg=dlE`Sa`$8G$JXp9ek!$WxGmOL02MZh zj>q&kSP?j1%-JI=AQ8VMq*~Y$2>3Pa$7$&?hI+t^{(A1>jag>Yg$!XQx&;O^Dy0M} zyapI*mJ2I}{Tnxdw2Zf^l^NUU#s3tc^S=+@s1L{^qDuVQ6SXKXmA7pj#TOMaW>%@H zjXT&`lx^JGTn5hW2q)9N83BLl1peEA(-5L^)c2ektAm3haJPwz?1 zT#MpBsV-&vWSAEnQWRlX*B6jvM5oPKr#VAy@okUwP9B1-2-i@0>u*j{JM_##VD(l8 zRf%q7po1zI-#b2taNoNz!eBA!iDvz!3ZKiAbFhL`r!DC{n*@=I38DQ>CYGT!2@ z%aE~IBr^>H2bZqviKUgr9kVYcBQ|NA=Ecdsz2jIS<`N`v+w=F18}ytraR%8Go?F+h z3k*I%Kyxy!x}6~RACRnchd49fv@xM0nGuxCOJ9L=8;S!MYB-Tz&E9R5unSl$BB!6i zwU@ucjtcbekdLq2?n*rpc86gdt>xJ7&vONT7bRs#=y>tnpLiR}B3Txn7M+tS#?TE1RuEc+^>b760s7w3Ldduqd(tusKQ*Sr?KgK z@pJ(&mR%vP}+IYhNz&iN+SS7x6bQ$6^%29uj_>QyX$mAtHO+6tOOMI`E9-aDiqe#p)q;B?6__sdbKP8RmgafAgUx+%1BI6`ThbJgW|c zCEqYt{mh`wZzjevG_;{PdpxRbyF9tgW=;pR=eEu8*duAs77b`>l`i7U8fx)`{#Pxe zjS(lE9O^C2djc9U)?&49ef}4Mda0;ZH9hBc!bmjDi?T4L(rB5~0_&*i^t#;frgAr~ zv8GmPH?vWDH!LlXyQPkhTApX@nW;Km5%LChZd$}j5D9dhxJF1V$3X;sL_(Rb^OiH$ zi3n|__FO@zx-HyEiEYz}Cc~T>X_f4R%av|PeU$%_7W|CS>UbbKM!u;u@xzY`lLv+&&EXJjcIHZz0RMzW^j_(ESI*)Rfiv@?-BvRByw|^Kcs?`$L45ma{r*i&olqq3X z@0FL5wXEYDXH{#U>hKW-C*BG8JA9@jgOnr%K>Ike^=u{Ldw#Ho81by^a%AT-RPwQR z+`)L184IT-&n-(|_mjUO#>Tdm)gorKf(23NaHM9bdo6?-K?ZkjLUfY5IIhWiPPY@= zhny_a@O~h+NwdarDc;Iec@`XvP=JHcqbsx@tLSXh|-$LS!qd^a;q zepQydS=Ok}>U&+8^qr(Oby&@^eR`)#8Aq~5<0oc6CRlb!)#=+rYjmK4-gU@b@P8`n zN&ESo@j>hGB?eRv=;xE?nSy`l#l7cQK!l~7n8{00AvWMejSN;M3#G9*{7t&|)gA39*q`YEL8 zy$iysh}85^Z|}Yn$-In&`|RT!WO3H4Xzl(1l8!R7WyWgw#KgY9>p;$-)bpZ^!N50u zyWXUR>gH;V>NBJL{<8wl*DjqsXsr+~Goj~bTf||5l{nz{4E#c@u#yzlqTO$>M{BO?Oi=%-S!@BUK5|zPb+}N=v?uT2lZD+SD z3ete|8j^Z6>5b&x6iRzO*N0F=+9gRf0!>5}l{GAoZyR(gL;6QM0Nt`D{gy$a%Ji`v z@p#C~QtzruOu5jjp3@19JBLRYum$Jy6OCnkAqyg}r@}N8qPUKRehKr(+PP)Ag%c-( zb`4Y^dnIxo#|IT*w)Wc=-Z4+;ZPVOOI>AdXOA~QCE=F(UmUmxYC=6#2Jviyce(R%e|@3ut|(Dt>p;H zJvxI>ICD6RSc})^-cp>Eku=$5KSpUZT4d;(vkAFuiKP1*N8ySjRwteySVmVAfptZ{N0h3xTrMO;uH=B6QZv7Og zTU~AHVLyuNe2tbN%Bb*m1LgKUaE8HMHx9$$hR6@(dT4j>!~su&B1LSL?xnCO$BHGn zoF>7+w2>G81IvP7RM;^yAN=yL-FI@ifEaDHV?TaHi_1eO7Q+D<%R+{o3LQfKjMN%D z_?ii@XCk;5S71{mIOo|~9h14*nOIoZS&z3DKRy7>X?`V%Y?<1bgmLV%v;D=_I7_btNNWRhDleqi z`d*zy4E@%dWST#ifB5xc|LWIgiPymz3g}ns8V}*H;kcB8lBDsU+7QriCSgy7TdSEn?3g$=Hq^2WK|Y;Ef4G^E4xS;1jkUv(@+&0&6^! z(05!u@Ax$O&Yz6miT8KyY7|W5~!Gv0*451#!#dbfLP})yl@pw zjUD7~d;Di;&kZ9*N^q8g?-!U2H94?W7i{YiQ&Ypzd0n5+NvAVb+bd^ss&{2s%D4cH za4!BwFZ18&%QK$se@k}$sURTe+Fq!Dat$w0b{+PhgGSX8*3OeYZ=z>xdDOpDK3Q^b z2RW$121*>wxm{%aQmEF{h%S_ zT^)LbDUB8e#LH`f`tS@-B7F#~KVYOQn)t=qte6h$g1;kg&;()Mv0~eK2_ghu^ohw| zI7NMS>!;0K9#atP;;ty^-rcd0R91EvaE7d(A`IArfvy|WfAqWmy|3GY*m;RF^MQ1~ z_SmcU;sku#;AexY?O&)%+T@dylbEEWAkgrlrNInoCj$-7CjyCKK6~0Z7yccNcKMp8 z166P`Kh-#%zIq{OfRBS?u+shse09s=S!J-?A?F8vDpNOm3#3q)0v1y-`0i5~VFw50 zRP%V=f)3EH8YxNQ4l61_6u88H##-)qW?d%+VIA4>EGBH~+*%C(!uV~+ozp1@25@fj zhZslkPhJ+rCnV(P8)<#^pJ9!s4~97XLQuf`8|RAX0%`TA2?FgM5b3Gw>VDLBnOgQE zc_Sq)dzk-*PQ9@8M{Y(a=*mg+y+TcP>>v(~jAZHm)s>Yy!dZZ;G@v-5Z2@}v|3P|c zpB3p=5xMF+Lg&-<+wT_CdF6;eiT0~2c?XpH*NVBqom5Ww-m8HmVKqQ!(lYy9; zS*1FR;KMB&WBnfrdfj>)f(1{bc_eK64j!$Z;N_(t1NX5H8MKx z6>Jx)h42!AI}AYS<5l04V}=-sUDa)7bbP|sqmkI|Lbmb&{RMZ3N#c){(HgLeCaRtt zDYDbYU6}u47s?pLS{3l1l^47D_E=_oVuF%UfMk?~#kxAhvZ-BeMQAdOU%M1TRDuh9r76XR-^5;#*-^#u7*wHZVH|KlZ6b22h z2Y4f!9Cagqf-2-CDMJZ;+Bf{!ryCuKe>r{~ZBjt~Ro_ECIf{zFGa|aWC?+U&-rQSR z`P#(H?D^M$U)3i-t3VIa1(bf`1()D<3H%>{|T(_UtQFHI3{>dxX&K<{qZ?0mZj93zZ>gVustT z2%}8K0T#_tj>^p7_Zd_)f#i=I^saHgs1Hea0WAY53HXn$ihwOkbcK^k2qCZk{zW{k z)bzZ;aw@yl4Exo#Yj)q(42&KD62fxfM-mAm9f#nm`Id#g!6P!{ zA^W#11I^(G1WX|=jO2%<%)=Ij8@$^TmhaRrBYM{m)CFA%aR%Yw;52LP$X_aC?Dj~} ze+C^uWSMGu4c6i``ar=wSsYP%8w7x0qb-VL??sF2tx_7~-9Arqhlw0&!g_<07FqNJa3LXoBl(4efM5}MKB$pFr$X!u*2DMN-!9((ah!eJ zNX?!^9e4dpSzUJ}7K>6J*Q*`m@O{^8a68W6cK%*N7M25(0C>6l`b{2m~>(Zx%ijjuTCYdkTW&a1r@2#SZ}L z_pww(lM3q)8@wy{VJ}M)*k|jZlO`tkqHt&6UO{Q$Eu88keoE!ghn2ofgZ7_ca>ruT zVW)533jE`D0r+#5-((hnqCcWC#ExcJ3h|IGKY%bun)MmyO`8==P;n{QvJ=N@E`L~A z18L+jb87v+`1;DQHoI-x z7AWo%*A{mu?$QFqOL2EA?(SB+xI=Mw2oA-e1b4RrE$$ZNzUjC3x##TX-1{d#ctWz? zwdR~_$`~WS(;>8@0JG|U^^KxM|Ar+_FC>C;lM}6A+%`QamtI>+cW*g<^l?7?4B3?Y z@8@0)RW6>cH@89dV>Kd5!VD|GX~CZ?LuF*}Nj+ z2-C5}{Jfj@ofqDH0Y*SCmZ?gK{SBVC+AW;S{zV*iN^~jgv6PxjRYww9tp$Lbnlr3m z0nFV57~8}|`I<3Gi$ZUy8)4uDh?sXdqNXmd)Q-LJtz_`O?j;gO`*(as?`~d^{kCf3 zk2zwL;I^}%QbVMF?1T5%IxOh)4KwGvyX8J``cR-Tv`EE-4uk>~3~)dGpaE9S9k50o z3T)3>)UG}ZZjz~n-YpHcigzH5c?T8~j|bd<`TEz2OR}TBIxt0!(^`WumrK!blmnNNapWe*CL8L$M2o( zyM`y-?$p1jLv?6~EKNKr!XQ;*hg5ApQ%yiYFA6Uv9U5RK?zU9{)(+T9EuKX@h)t!aEIbeC$03frq%bz(sNuO&TOPS|FZyK|=o8kQwPq z`ejgJRX2}--oV<@sF8MwS>Fe0W9PXvg4GLDDjIiTL=)Z0o-QhYHgVtVF-bR)$2;(# zjJS|XmNMVIrSpbv-YZjoe8}A_aI7UO=4! zUl>|p5RMYT$ji$wdSDn!oPIG#sLJHxy#6Nolc%@DauoW26)e@Idnfa8${U*{gJcUm-FBO)TMF`UO{T7TqG;rc2?-XBMh5Z+tQI#5G4 z7&%*YHx zrbfB4lU8YVpBB;ve_&0^&kWWBqC~Ll{U8&+_G=q^A{&)4ViK+8KfeWtV{XZ^U*WW* z^9a2u!aTuDrhCu2ys>xO{B`2c1*7bn* zxpT&{+wRoRdsx3>6+Dc=&(!}7c$!1~88(n9RRaOpD|)OPg+8^TXCM;h!}rosERka! zAP}!(8hJFG_31cTV?-Tr5F-?kMq5OfcHN*t!8$051@gl&>8C+xN`!Ao5P6AzhS1qz zid5<|H23))lw2Ai+K2wc5DQU=~$_*yRI`A8qFh`x{%Bt zZJF1eSV{RGus;l}r3|tgRNV$rFK=ISRd?B~b1d;!M{wm0L#|m8LtaH1jE#>=>FJRb z$!F|35U0ffS^gzr`&GZ>17wdse`{LOd=osm+J7k%LvUA4Tln8GZ+;<$EmRvE0j>A^ z6)aCY$tT|OhqX+?7!g;xt%Dc?acL=={(A32+>@<3PfvLOOZUUU+W{)h7in)k{y=~k zl`V$h($`~frSz=P}llEgxnB6$Xi^isgO%MJ)! zbPHt)fP0VzqDYRjCNRJhHHQFL(LT7Mmci`&@Jt%=MSp`lBz^6 z)_41QnyLY~uTBhnX8k56ROlt+^S@Wq#LB;@=din1CH-8Mc zK!8Xk1H#DqDJxA8jQC)_qxRsV+Tz|i5P$=3z7a^-s>}wdedeWJj6r6V635g zrnWA<7nkd;!oN_+n`{EOqa&TDtJ99CXNOLhcoWos-{|)2=mJpXljbu9?3zf?fA~xX z=msR%h)gfvv-#;8MzJXpWX7NjAa|3DW10C~Axot`N+j>xi;EuD>U`D>u14N7v|4n` zFQJn{g?A>7HHPR!B=)K3N9;8U1J}j%%k1Km&um^Y5qBRc*nmq2wl_-&xUK)9s%{ki zxkFtPM33Lr*I(i@vHGGdGpa?HDrR7k;)cOa({2nMMouu(cU*|4_Syme{ul~sH26pI zgMt8wvpSnmZJforG~M`@lRechPywtViTg$X(DbDk;-L8-c}Bw+((9zhz;TqPD4|A? z{L+?}>`G?@>0C(0UfGP2$^GJbJqh*~?p8`#Z^RT841q)9K|@p?746$$Vl z;rFX}k^iAmL8%X~>Iq5jKT@_3f(i@WnAp4|v-ZlDd(r`q^~F{v=kfX(Qwc)v&jz*T^(9%c}O1xO-AP2+Um zPOPj%k)w3}a4WR?-mWKzO@sF!6Fxfd+m?ep@5I4~+?C~ip8C7*m|XXv`HwnxAnsFe zz##<#w257|gHPOHaB{%nGJc!|MFuWne9|l!gaeRjHD=33Ow1lgSMo*zd}pkDt+X*m zeE6`QF(2emiw(U{r;7mmEVn;UMnWxCG!-a1(Cg7noj(46A_I{~M4JEM7J7d8UIZd4 z2R$N)s64?&kIov!?$8zV-}KyO5>GTiFk^oj&_IAtxBMeBa1me>iNP2`XP99A#L+h| zaR8I0v47aeGq{FT!CRyc*l$Jl?(%v$_u7w6GMN#rc7@WCF6Or|@nf)tEAK7Xu^1CJ zF(8mnbb~M%+U)>C9}LPo>4^ju)$s@<$UvU~mMDX$=66b$f%4s3_BtKfZ@k~Kcu59- z=>;4%Jn3vO4%uM?*t-)7HGr`~@eWv&*pcoixz_v<-#$Bb}Lhoy@ zEa-h5M zfn?u_>hj(4M;Hjyro3o@ml1)_`A00P0j#dVVLfGyMx=*kwe9Lm@U-&Xn~z15Tzr(D zYlBVN02!USWeuPV53q-F9FB#+*KvSO1=PLNG#zua`Fi$_f3J6mKxah&$-o!@svgx1 z(brYwxGH2*1SlX5L8>fH)V!6T*?A?7BI5`pGUey-9xR`+_eMsaX{jejt)UNGDWyRU z++fQ{yyZNRXw$@6+caqYlngZx3IHYRiolGG2ps{>-JFU%ond%iPDbC&}890(FrQ_+RJV5EqAc=Rw0r!l^aa$oW>+2 zT}|3b<@pHg27QRX;8w7 za$ADwz&$wk5LOz zMP1!^ER*kB{;)3J)b#Wjz>nSn#76%7GzAlH!ex$`RWRyAn6~MCP*tuKqvYl%+;*z! z|54G4g+FrRM7RD{j5!s6GED+7D87!|(w1T7J-XUqH)pvh4Hp(7cJ_cQ@jswIL1~~A zxP)FI|DcdD01O|BXJ`cl0FAhT>(o;J>LGjm)ii*F_rYI9$JZHyiiXNvpQ4%*5vns} zp5=lEV5@~_-Wm45dEah^eWdVwZ|4J8u)%} z+&t(=utFh5ow@^qrpmQLSI#8X=)}s9&SPAiI`v7Z356@V(AHs4s+fXHnAp;RZ|JKK zB9eUCM`>vewVcTC7zZyjMPQ_>p(x7YsHDICmElZB!$wz#q5MEAaEKXR*Q)xX03Z8H zq_f(OGGFnl1D@x&G+tc**+vZj0Y~0V76^w&3Z@OH_FGzfn>u)T63RG$)Xn6pczzl! zYHCVt+i|64+^2#5!`T&}I1qL@(aNXKXsX>y|DXVB#j-88v&X8N$q60m0>{_yaNswG z4i|st@C6$2$sH$r#{|2eH7@A2lZ9|uDJ?Z1nO+HeQtCpHAG>EUbp0CrW2YX*@~q`! z>``p?B;`cYAf~hIuc8fIm|-y0#+v}M+`thAt3F5*iMDH!v6L>hj8&n9t+=K9K?Np# z(T+Gn6(uu05&hZZ)<_k>rEHmaPKC;M7dU$@D+caGl^ zA5^F#FE394g?}>`=M+%;py1qFbCrD~6q^n$8{rvLoPfsJ6XCh|B zA9|rUlAMT3TOmM{y=yPj>Y(h*X0uI0w@wrsyvp-&g5!??Nstyh8`KGQNB^ebk$2j_X{qqqcA05iqi7+e%dn#kw) ztvmc%Q>7!4^D zNbMQl;dq^qA=g9ZUAvjqU*OK}ENC^NuH&1OBd$l-OvXIX})A@%O81YUbRhHa(2p+L9;hiUq(RC z{^|13xhZx3Msv`V!^%sCA-RLy8yf<;)T?X*DDPGQh?`G^kG&Wo8zuQM;_~v*%c1~- zZlHe5v9>xY3bo0@ed-zRm$`rX(wD?Y%_42al&!sZ38Th8FqAb17Xfl5plt4o!hx%v z?o~wzg_So$AAgE{&U52Hx=zM5);wceP`OU3h$hKsu=@JoVZVM5p`3@NQTAs((*^+_E=b%!(* z>j_>X3}?O94BntpFoLU0Jwa*{b@L9GoI_71}ewb<7qw}t1# zK|!#S1XDKH=p(`igqE7VhP~pjia)r!b#y?a>$ckdfdhpDFnT|Ks1PGpWweY+N8+%@ ze7KY$4cPoMZSQ|3+Q~J|!Cyx6%1O$^U3dX$hKJj5SW)8kixnyN309;B*H&hp6k*FB)_{J~z#VEj?E5K&Zl#v`xcC+M@iAut ztkX6C#<-80WH&&&{QR}-UYy-Q>;cCPeLl=KtMKorPHO-YSJYdDA?A<|rPni?B0J@B zJz3>7cObGB*P;(Tz2fQbWE@`6BpSw8SKSoZ^e3XZk4fnv1YIX3XHuu!ZTl<=* zj3`3;L!EDLnFTPP7S^}sl1$XVGceh2AxIJ+ETw)3QgMRnlp+hn5d zfaIt1${z z{rC$g9K0Wx+6r7W!g0Ujr%-PeZih(x&fxm6f&l>)CH44f-^FS)Tv^rzjVo-&yVrg{ zeM7iskche9;P9j!(kB0yib(ge3&2WVLe3vTZd+_~Qv-F4JUPnYE{Syk;jRPry!iMWiM^qqCyhnT|m7eFjF)?73=(Nzfz+)_oY`Ec%EQlT?$x>X#xPDIp76* ze_4~OWu7IZtg}DdaPAAAZSWDIOZA~1IshKm8!D=bA~arI2`JM6^da-8cO!(~!o$N; zMf^J7KNI$~|{*w<7LGl3J0Y}afx>Tec$uNOW3{V%ac9&P`KH*UV36FO9`;21OI{GgyQKiE z7u}wIFp!$!L6c8{5^0AVpKHP~-fdKX5VAZsl5Bzl7DqYH}0H7 zN_5BsYpC4}TSEB65ej{UKiQ-KtR?{T&K2sZ>+Q@hS)3CFP9*VeG$#8=>`KIyByda; z=f#db#xGq_yah?Q`r^IRE&s!@TSr|ztT1D}d#S9cAN6?-i{VOA#fCKYU(8Ez==mdBAk(%HyrAkzOCA@#ySQ^}`>w7s zyETgIb?V|<>K^*>gKCjsxyvW$c3c31!(;Hg(%rs?8|dL3@6XM?=9iYKeB8={pW}Y9 zWJv7+Bd(r4IbcJ4-(toDwzRe`VHP^xSFWIFKItMRZvz17{-s7MhiQ2p!?_AgFEA!CD-?0uIyWjop>CUj|8zAB~JT~Kq8Vsq01lk=wSZc}7@`CNG zoqV5#oy&P@z8uVQ{*`6R=*R5JW216FjuA9~k58!Z_$u-#&|^GXXlP_4$lJ^uAm$Dz zN=r+R7~$9XQK8J|0?>>dw+_z(C*70b@_)glRHn<0qFMbS-kBi6qY(uIGC{=3 zE%2xW27lhFd=d0pb|kunlt-LR#!?S#>6x8i0OD|UF@HU~^k^EeEYihu$e-a}4*RhM z>knRc0fE5Su~kqJKp`Ro*aM_slzWLtJj3q5SGU*8c1nP1hHjfHq5i_i`q}x!GdvWj zw&-;*J;E;QMf|=);oqWrI-LLo(8S!Fl7|OBjMO!?w)N)rY&$OVaJ94Blca4LSJm{^ zb8lpb71DE*Tvv6U`j7ur1K0R!+BmNI_#XcWrT|kh?WLHEvl~e~AC8X~s%^fF&fP81 z6C4vR_^wUO&|%Wai0d$qo2R<3WbH>;B_;R2w)R`+ZK^Fn5ms&&82@S;EnE2rYR~6^8uQG4$C%8 zn+?+n^}l@oFcr-{;T|+?@8BkF`Lx;1PK>RQ$7c!tCF5PSw~o_U|ASnzrS0QoJNQaQ zz{4vW#ov^*Rk-JpVyh>FU&3tdz$Yn;ASG@q%a1t_fI#+DK7*T(oWEE;1Me#X8x2rR z$;0(Xu$?T_!U7oz)6;cRm*L)67PJ0@Y}Ed`E;Jsv2KZ(b3v=n4gZ^a&P$<}ZA*H^8 zLQHMM&3g=?;A|nU{ghWOT4QXtG|&1*yMMa0$fgfVddzZOs^okbsrzfn$_Kgnd*{Pa z+!%-4FN;Z>I}Tp!Q-1R|4Q#3KUF0P>pBD8egOVA1yD!$?$h&aW(z^Q zPt^E2%_4JZ7XU1PGV;FpHRl-{v2RUKvTfDmvl;yECg61E&Ff1v@@QV#-zxe4S#Zd5 z6NTTSR>h+&p&H90OwvvvAS2id=vo>x(Y$@$2eN`QvEk9XFM?}JeANkFfI8*T zUwctBEz~c*2V3)5#7#4!&_W)#KaFMrPhM1p=}xvH+%`x8$n<7k0es_V9qd?@DZpT$P!1B*2x z8Rq2&G6Kv6b|dc1&hd$|9M@Z%nh$LAGiuuaWDlup#^-omKUB>nY%xRpin`Ba103Lo znUMwoIfAkRTN)Z>H2RG0&Ub#V+Px^!f3Si=6q$d|Tm$0E|7J1%C$iE;Eq2VCu}d>5 zCENYQ!MryDpQ<(s&D&zdke*r_b$Y!!GYsydsw92CCY8mgXn+glQ43&k$Gn*X8sE)a zWG_8(;Mo7cvgD=Uxx}{zRG1s)(_-Pob3{u}%tQ`#W^>lS|!a0vr zH&ohGF0o>m&DaN4_gEDLq$Y*>fA1^xMZk00M-*a?0`lARqs11(98uokCn9=YX|jzF zUkvuJp1Mq;Y@sQUsRe2Zx7pk8r}4UONPJ!FBG2Ht3DpJ23SJ&Sog1Aq6UC(Cd(pvb zIF)9UZhugBv(JhRSQixS*|eygqZgyOv`heHAsRzd!Ez~c;k%}2+fKfLYaBq;4HgObgaKAwDLmUhM^08=*VFGEsFs*}Ig;y{~KD7s7d=IfdD1!Hx)ksv%C z7$x9nr2{w8O(;m0>!7i*=1Q%pNZ?Bs7(kCM4F|gW;hJl25vg>`2HX? zD_n4iBi=HljxfsBxbNkh$u9Do+oCXbAHJT}|7`5!;2oPWDdP5Zz9|^kO<`6}pL=H} z?M@iwu?|Fa#O}<*71zyoS1D0nMh2jTDDiQnBI_X2zhKT#Wzme;UL1WNAJVnQWx@9A z2l7cu#_8J)sFJQC3z|l@e#FnSA=+0Zj`co?f48vi%|BNc=H-D(PBF+k`>u(0&%nSo zv^WLMR2X^p`yrlM1l)r-MdB+b4-dYo)_n%3<_3YwEqgQMEalr`{-YDHG@Bd1!cAiglG@os8^rc*gY(+3uv1Q|?lv^NdD`{&7QETfBw*Cy+O1SL2 zQW7jJ)`X}60o4)sEVo_dg95;_bwKRj=6tYY4xIR;(sVz){WVJq2<~bFb0|c88!x4H z()2#GW&V2QMrv8W8u-osjTCh2>?SPUYF-CFKk`k4~ao0;tUXud=~MVf2k+Y82o&><|U4Z z)bTMPkAf6MLSpAnz2LIfb=86w{;U2#U8O@@r9PO?iF&X9aJ>oPobuWmJse#FudW<+ zw9p4Phs7R($1*wgz{jhreCA_O?;H&6Re+)t7%Wf9Knw;FahU=y%HJtvxZ4&M-<`e~^ghY8w{V9E z`ew2`lv!&Sv^w3W!C>r^8qgal%O%%K zdnwZ)Pi!+y=wHY|7CD_l7$|zaL_oh6ez1HqQJJ*10PmzP79smPg}&+Ne)jdWm|8L? z5=U$wwRPm23go;UZ`WCQ&p9VL^@pBTe1Mw~TEmdR80%Q~IKATA>bHOx>qD+=YB=!u zYE7P;%+>kL*{D{Wtn&s@XdSSt`3Pz16B(3k9jLn}UC(4Gx~_!n-jD3&-EW=$%;hWb z?A%g;qo8ZU`p?zsi5wOtCBNRTVl=04I6((z@F+US*EiXEUD(L}>~5c3Eb<;v3qJTG zIYk8?{RrL9I+P{P_&Jh=enWVz^%W*L;_mG05FyP&Cx$*SCdnBst=Fei35 zQ<};FzFfru9$R93g;HjLBKCq=vf@e-K)Vtkd7Q3cSYfT#S0_t!?_B8i$R#!hH=^;$ zOoQ^n1Q_1WC9sXe1Alp}k*{?f#7A)g~KVS{l0{m<1}&k zN(BxKb5mCwMUlymEYDiK9y|2FU+GVlVvY@PV}b7FTIvHX=$H2gB6gd=M4pUbO;hX? zdqdk-F(h>2KdWF&Xfw?6V0win?404=Zux8wZ>;T;w}0-ucI1|bydsXUTqUQ$M^(2; zLJ#r-a<^u{9WilzY?G1btOtl- zOF5N+i0zc53D=$|NtZdwhDiHbHH20_rU$oBhs!SD$zO|WD^8`uUQECb+-~hFi(~%s zGuo}>MBUxhaR{29_z_+ikTvY#ZYU9I#C8*NI&fso2FWCVzlb3fU;~JW~B|0_RoIu0;h}NP?QW1PHxZnjbrqu?UtHPB~CqUVQVJRI`g1@K|8k zzv7nw$IPkyVY`(g_bDI%x7R&%e_m3yEgV7!3{F$s!k**arjuWeAv-QX)l}?ZDsOF- zA*Ae;+xG6rG%9bT5nC`=Yx9Z3?x@Qr0$aI}3`i~iv^=RB%g!tEg`?@`1D%M2=nZX^ z@r2TU)x)Z(r74E@4Uq*@3jZ++yTz*~U|D2V%boPk4Zx9vr3vE867;hONkiL3!lDHu zUtAw8k$0Wz3Hr@^6qREU|Zss96lPITx7$DfQ`|_LP?LlaEmcj+75Riihs>OSOc} zE@-^gY$x*T!Lyo9%-6I-ZzdaF5G6{o)$-|L8zf7`#)A6L>aQTXF|`h+MSEV z`De=zum1E3|C2%%xB6YxYU3}OCm@|0LaA;W8!pX`K;;LpVH)eB5ikwoJO9QUiF+^W_q_*W$7iC#gKvGEOuTk0p z-vg3J@R^)T;IyhKujf%f>BM<#jp2PmyHgoG%sbxzPx3BxkxtWNLzje3TjoLvjpO6F zSpr*R%I!-T^ZJc|u1|?p$o#)>e#c-b8(XL@M&J5R1u_)GWw@GJqS<;uRNU{HxrAU4 z=;~OP_B|UY_ViLGZ;;%*T;=c;dkx@`&mB*Df||Y;wZcDlJifdnV(aq`XC;{@&(>Qb zaNBLMce?qcmiesHrKM@4CHw5)*9@6OD#lPUBUdpRk~4= zcJ4*Tx_^Om>T+Cju+5|idBYXeC>HO~NXT0=sW%J_#>IDa+}ynepd#Gu%-n#n(wDsk zw)`Uz?OP8qC51wDxI1Xw1v>ZTE;2qq;TcJm0 zk|BLDfzI@Wi~gVqdim(Cf*R~a`obYWjlg!GD|da0axL1kVG#VmqkDOL)Vkrz@;AV~ zS10lC~`UV8nH0iDCl^QY# z&tCe0-uZ+mssJPWaW+dyxj^XtPp#LxDtYbAM72IJ8;6Gr7~@Q$uz;a{#Cx2X;=P^! zRRWphYL88x`Qk2b!?ok>Y9084Ej6IxQbk8_l5_g;*8#cq2W0c07O9+s(}@)~3+#cE z<3_Ct-7}nBk;fOuo{nUaQE4-I2MbH^7gYsv56<6_i~Cv)HN@!zg-F_1-9Ncp#?`vo zajz|>E}PV_x}07=2{(9Sr?53KIv|z5o6RSa-?5yF8}J$Z{W0nf=~TrR(g46V1$tD0 zUwX9o?!#$700yD$KCFcLN$3iHbY(zdm~NSo1Q_sgxMka!*Rf9GoQd{ti8X6%Ga6jhqS@`ml!utWLu9AbuuihrAzPXzt0cwBUa6}$ z##IyIqCY3N3#FVK90xJG%IVIJ8`Ms4Zr*FQD5hcPr?6d~=J-=|ecE|beq^lNY{2&JPmB>_Er*kibk&{T~^;g87 zOuyW%=w?muOd*g}jYSlHgm3von!E1x`qe4GR#VQl{Us)T(t4OD0|*Q^(JVb=*2p-Y znovBd>^+s^uQ>a@cOfo@-Jv%_QlRwpPjJg56e06BWFyC@Bz}j6MneAAEGCzPZZ{{` zoiTES(i~WR>#ek+&>jM%#TSvv}KGKE8Q?vF@)gN^y@y7C_IzD&d@BYhPTc zsv{#CQDb=Z?r{Bz=Q=Kq-*Eq~`S^H{>0JoZ$enp)s5uJGjwo~#Alpzco9IaIJz%zRCkfvGZ+YgWoS?r|E@(i1>OA9T%@SAfH(&C$T)$Dg03MBz%Koa-b(p`CX@K(S} ziPB{#L%s)GvLz^cx%Q}CYpj~w(plcd(V_=2{*j7^nnn_Rx&T_Gthnx9dHHf7um2Xk z=l1vVum%!NqZUFUJV(<|TuYGP7S@3;@_lfu7$V`AXFpr24KY|jVeBVL!fJjQP}>a4 z3~DS*3)<|O;<2g=3Gql+VFaa}D8@XkP=bjmTbm6`suA5=m-Ue3N0@uxuak!=SX*Dz zSUHelT}lcOR|Pj0qiJ{* z5B8&Skv3ny&#(2WcfGYSWm8vI+uCl@>nXPD`uhEwJ-Gc`QZDsf&{j0(m|9Oc_|+ZX zt<^SB|2Hxr2V|W-s?slPis`~QJR-MDm6$x%(}p|;)cVYQ59Y_}q1RJAi*V2j6mhP8+!0kC_PzytCJ28cMZKkr7DP;aMo=8X1d z!Cb;k?>-VXUA^y879Zv5O3O`pv50Sz0J7-LJ|iG!N0~IN?7A=DN}cVjWo%B7MZAZX z6U(VNJH0xmb;}*7;;1xbb7GM|FYA`1Zlh+`Q_hJWj*f4rCr)%$H_AHm-B=>RA)nS= zT}cc>AXYC6D2$O3Vz}Bjtrg%uQE_{d#!|(iGWa=E3#j?;dqecUt*{!w-$d%iED~C_ zdZqI5T>X}wPJB|Gg7w|YAx^|U!{u3z6ow6BIO4xA?m#u0qdgdlMBQsPaLb(Kc>&uB z_aiIMm#6SxG`z*##if@jX^p_<5nhE*t21Mu-@ruNe~Poq-K{8vqHh_Eo%29em;1(M zH5aD+uI)-lVTt?d-jX%C^Vxp21T*T8aMA=9NWVUF!FNvp#e(aEs1$qZY_8{2P`#`_ z2hAnwj~d&T4Xg|K!*RT%GO^LO3b>)fb4da{aG6YFL71<6-n#cgGxRCuCsSmrm(Q87nBEo#h9xeMk zx-OVm+`%EAUNkbjr*GcparExPpXReg=J~;Z0W17z7Tn)sL;4)_APo83At-&&O{wKGavo_HL*z3{MhNd zyJO*U5czNkMCezg`%2Ny4!P5T+Y>{OusBA5G~i9N`gUX_a*fS2Kh^QFL`!7T)``5; z?|{9i1gTi5F0o$ASvUSX7iS+{vGq^ zPby}Nzwy_t773+Jm!@xWG{dQofr;29WoHnjGQ%$>gY4ob??J~vmh&pBF(ksNm89wh62*TVPcM4`oc&%tXahIs4ob+M=iKv4h~*vL9hwFm?=+2{F{ zt%n8oSlc0yunkMr5)hAjt>2}_euhUY?Pa-)7Bn`%^bGUr8CMr^{n=T;rIF3OT)`^! zQdQV)J=5n8IUf8)qLtUyjFZu@)`MzOU*z!L+b(L(wtqkq^l*s_Wxn}GZl^S%!QG%%It zbWH#*^T(_v`*=zgd)!Tlb%_V$A*Ac5{^7Ru*QHMnD@DKD)V<+p6O?Bj7VfA!e4jxT7d49K4 zG!W?foWcuJt25f^>t7ZNNw0PKmb}e(c^>Qf=0WwUSU$U|Sw2K6&hHzYDT)1d$lz)3 z-U%a%s>7<+WJFWsJAuRZHgc)1FNNKA2n$1n;5j&0^v}8Qz+Ujb8$hMp>9W)3HV4x{c zDehDcum;M--0}IsUH`%%>fh> zdV*Y@*g3p7_4T6Yb9HF{`b~;HEWy*L8(1K|#^6gY4rn42>c0oKIfxtepNt?7P7*!gCrH)CpY2BNKmS-z4B+DgFne zE968qQ%h+z2Yho3xK=lb_IX~^eMkPzhX*Sihkjg{FTogoOBRhWBssxpX%4dnPe+4x z^!5X&)8ff+QeSh-*@7Q+bgbhz{gpNU{l>qWfZa9TsGwCDvr$W?v@u1e zW-V9K)HQbNQD%P3GqF5%gf&uRuzk?%4Fwxw${HKWd&$hA6%Inn;oG=C&j)!(3i7!e zhK3i~u6`#3mAvy>Fxv-$(no#yZsZHg4|pfNLL4(aAL4;nqP1rJKPmiH{H%t;lxLo| zAC*W!=JVg{LicC!kGPb_Gh+MpPc{RO8@2=sV=+8kkUp>c*3)Xm_cL8|>)Idw#))(j ze%e#P(b|)>@~5SwA-`Ys_P9Eq*SR6ng2paR<@~h7h9JJ>pf0XcF^wB{sk@ajx{TADR%b!R#U3)FhvQ2oydNOnxmwOD%=6@qp zcYQ$meYMNOI8okZ87J=3RAL-2C6t~G{ASTu=i_f?CigqS;9x1k<3I_-SkMFaAsF6C z(iJ`X{)o;aL*WF!by2}s)^ZBXD zWzl_mPJXv&<`owRBd-mzd6)tD4iQyua0C{tqq&96&7*_yd$OS&doOdkL&(fv%D;Ta z686}H$O6=z>A^|5PTqnt+ejU)^2UxEiGyvFj-oPB(Sk#S(NIkN(xu@k)K|{zYL-vqIc(bRE0C#y8pW@!0JY zi5v#U3PSK|>rJ<=;%HDx)1OWmMIpHWVX^}0P_kd4SDAP1Q=%NtHQk3K9Iu1SR}5TY zTiRKvz(!P6J!o#~@1;~8xif3g&xHECGQA|+r%+hn)}NB1gA$ilvh!%)Pf>YaOXu?X zeWbI&Vk7LA8!1-(HO_5r?Bs~WiMU8qaC7 zt87kuh_ND%7o*yS>Aj{44t)LR$Y+uu4Bgw8lWV+f)%bbhu=KuB(ob8YijeS%GaXZa%nYVvH3ZW%ww*VU;}t$Jww||5xV*rJiLw zYwSluZj*&H{8~8YhJUv=T;%j|o`q*&^b7^m1&z_D$wk`BUo@(|24x4n_JFix99+a2 zO~+bloDT>m{>r%igqnjLba8x$@Jykcdh58Pt)xKjFz4+=(B*57`0Vv$;w(V_6_tk8 zlUMmW-8=Mj2C3WcugbKJJcl z&)*vCc-A1kXj@)a%I?1wJR0xUaV)sfqqYRdsvM6oU%cS1mHr^6;)hinBQl+rRrHOk zf%R_Ds|slOhHiB(+EFkCs@<1VXrFQH3JFIB`lxOf#`=n?GnRL^=iHb5$$#xz+3gMS zE1HgQ%iDq(V||a0uMRL%^3tc?=)CtxZM!_mKn%uhNMES=5uNNahSl9h#^D-XV zuTRg5q#qD0APZc0?0PCei+SQgJpjUw44Qh6EnA5+SdQCdZ^48eH2zGW8Dh?<+G?G($Ux>0L*g zUf+B@L}c;yNp&0gMF-BDC4YK@7RkEAO~qo^=yO#frRXnL|7eQB68$y`dE@gHgB5Ee47*=gZ(#ORI*xSd*-Uuu1 zXT)}kx$~8b`>n+KHo8!@;ykT8&9~@JX<_;FrhV#p4VyT>t3H5<-3&g`=DAS#N)I-$kRYRDCVRx_ha(*HwMa zCb@xQ*1X&NB#SWwA$ zi5bEg!f-4Z5X}q!;eFf{6_^B=y}!4?G095K)-Q$guDhu_-FIVM8W-fBBYdRSgID~3 zL1f~20@Sn*SL9xpa?9P#r;s3_N$#gk62B^K8zQyG^&JuKki0;_r4#Sy_^>qfjM+RQ zFj5C(JMqkX;wA{+C*$>}A5*3u<@vvIjJJIVZ|b2hujt4}4O%=G8uQ#a!k@e(^)*vu zrBdw(jBi~nYb6!dn(kQA*nbj#xV-kY)=x6!PUU;Cz~vP7rb^%c4idCHRJbxq+@&Z1 z%*UwJun>^8^X7pe=h1s*DWDf5!;!V;U5~bFz0~`#9S5;!WT=kA`&;9Ji+*jP&=qU# zcfQ>zx<+y=PLjgk*{$4|-Nv3i$KNv{^YK;BbWp{B3r@xgY)NhN+QZ(-!?x*|7Ev1?gs#`RF2>66)6F)P zRTO*}js?TFePMGN`xUF__cEvb?(N$=ev|aG6@?FegwbG+_mH)F zs&bAF$U$;Cmij1K%}kk_*>Ut8ZuFy8<5OjaD&#bH3ALF7dD28C&gVl3I@`ij7Oi6X zy?!rEO*)H1IDLr!IwZYe z9*sM%`uv0?ZW#*qb-l*_)pT`qbXA~xPf}&?VhLob0o?DAO;V!aExyiH^Vu$GZgb8qf)rFA3 zD!pU;9@Y?QPD&`wi8FXkzsExi_2#6*`eym};gHSG)>HU^S}Qsi$9KuC8{hYSKwk*j zFaIe!_BuTWl}cak`}%lM1=vT3o;k6L`P5{MWxF`(zngq|2$LUj-43l1|x#^w}g zLR@>~6Y}pFNHqp-HQs(fJqbpqtPEGI?Nq7OPu9UQ9@BW=LWc~=JW7;2k;0gbV=HLn zD{*lutH@Q}GH7Xa8|>;3sacEggZAW6kUwFwMJzycoXiyEH7V1kSDgUwaLxdab+#7@ zfaaARo+^}i-21ir6a*E;x!*gpm+AT>jW(tM&Kd0Bhk2y5jDA z>IK6?gHT4{4y7(^@_Wru{y<(_I+|fM zrgO0YP^tqIM9oH9$qQvUn$-}y&;yLkAX&qO_Iw_}HK9Aysq^h~FQr)HJInO(Dv^Ks zO}z1BIoJ2&R#xic6}DS7DFWYN9}8RY`8gb1BG&$E1%E~(X?Vj@J=x+a%S~K?Aw!of zAKOdIuVQ-> ziVEP%f`x?*o5J&y4bfds*Ax%bYB=9(a0tH6O+g(8n2tRw#1U ztA`3DoxplS2~W)8lX)THl#iU8i0!MDVKrdXpF7qcbHLNN>E3w)pah1AZgbZ10^3%~Z)Gf4~cN>^I6~*V?m=lFl^Xi$rLc^qQ|VKL69Wm%&6C z*=IMf(N^*c_l@% zBMiYyWpg|WKKLUMp+*&t*Oo-nbP_$Jp~%7=zrQm16Pp<1%6z1qTJol7`h_PMUc+;6 z(UA}jXXxLKyS;?4L3I3G7Z!q8UCG|iF*E_^cKm4<(~9#Kk1(Zgg|VCp>QV~y#qyE+ z`j{#m`OF?-Q>!jd!_{uFjp)fegCI+DT#E{Q>Nq=(2xWTDy8j(Qbi5)5z6mtbguC)1 z)Ha_j!SH9N_2E**uT{KX;))(5nc_A)xf$yHVD5z;h)DjH_KRk1arN^US#LnlWbCPm z{%PMRR^O+J&RwV;4Gjs3oRymmv4}8nRhr3+bH)r%=`XQhGy{GcIM>s-gez~Nus)cah z?HN8eUA-1w1e>Gm#ZoCW6TGh==Sv+&KrT6ypW=4A{=VsDJklbAMzfZW6%&<`KKA~( zj1^x5N#AZw-Ye*A^Q9@&J!@5y4AVkh0$Ojy?LcnvXdQm=9?}rpReJNVhwe`dW!^!hYpyb1!X>{UKyeQ=44uE0e*{*y$(+;M z?@hlapk7K#sbL@REBB7~w`4)N0E71|#>=F|p`WPI*Otj2IW zQ?9&PMKt<01U+j1+t7zGo?Ri-pA65maf{@zf{piM-d>Frp+8Gs9>xPtCr6Q-ZLS<} zo*YOkA3Z+fN<7S!V50{=kMRyXb0~r3UT(|$yM~yw_ z@3@Pm%h^WFdtWz-VcJm6OUG&6HmO+4sCglz)D(vp)02~t{Tj;&7 zs7&cm@J$CIAD?ovFmj`1Rz~aR;^QvB;yHDJ^-BoJ;7fjdQbC+yZViU1S{JSW1gYc(C-iM2ew0ossj4bCZ2GLq%oBV&()8KXdd5`jWXw4em|RX;3-{h_T@7HG(oaKHj8UbH+-e>A2Nb@KMs^-Uv)GBZzlbpybnY{DK;3M zMkwWpiCoQm`%)@0L0(lTcfke(A8_VO4>hz!(M8h@m7h2&r$vYhVsqvGtM?bpgZodUXd6&bz ziD=eME~yTTcd%ob4Cbmu^yVZdC3l5)&CZD7QRmg}bzBnSr-A}md}``P57XiUPpD!j zd;{vH(rag!W8m`?B$09x2*yAFc8$4J4$g85hlLBtI%RPY&V&C9W`6CGN#)r{v#RB_ zcr(LuhncQ!WV*UJsTgl{$$zmNBd2Zgm~qBmxJcuqk_Tcw{Ow? zK7#UIua7PpppaNxh@Y5|eGKXXfTBHGV3Dg*X9<}(JWfty`bN&N=is&j26kGCtY;Wj5i zT-y9IA3wk_yHBMDC|ql*#Cxdk1z+SAHpqvHKS9US8X;B|P8Xx|qV@B01@40&Q`EcPjogYX$>F3GWSbrNRz6P zI>d`&6#o9H2hRZ8o7~ifJr4tV zUqtzl+#(EBXr_y!*$R9*ZX3|KQN#=Kxtk8A1!PVD^w*sGrmP|YZ;2&^?8G^-b>`e? z<*KBWm6EcIw&5Nl$+_P>n%4?v-agm3&1fOkMFo2U$W2P;WBmy1Hz}1_fLGX8TF)S9 z_`dn12u&NbRD!0M55@Y~d3=~l^-Xf)D9i{Kir}8%A?JSSObdF@fqR~)y zrRqE4D9qO%uf2DUzLY7XIP$J&5~o4@!Wl9CZ85j+&Z_+&fNOHX+{HIshdgm1_9><; zKHZ03wNAGb%;U@S@F7hx&X2>#k9cYGNoxf%*lzE>Gh1SIf)aHLyfxNGCK^32tSTfx zM)G&bEPJy_aoVHO9BoVga{^TL^9mr30CG7_jCL+J)jB*vVSw(d6lr`YsgIAt zV+d$ZRiB4U@&!OfrwA6tk;Q68`}4d-y*cWcUx`_N7{#D)OqEZFt!0zoyCLdUIuS&G z)mTQ-2suv_>@ZdbD*C{a%c6gTvgEt1Jd3B1tuoGA^`qEg9Y-OK7A^-mE6`_1`Tlr! zt^hHwDDdHU+iAE>>@p|p|!>b)VDcP988D*&sRAlntU=H@c5;&xU8+S;Uu zzOXXI$;mK4K}F{KgZ)5sv=F9M`So@7pB>I)Zd}96Ahmq<43gNiztA_iIlA&w1 zDyJWn)b1*CG0ZeY-BYYl;Nr{NYV`_^Sjt?1G6R%&9>LjYl|QkH*{XG5)xz9d98n#B zI#P|9`&~PlT4`Akc768Ltw+%Lq~`#cVq86bqtyzt$9uQ>jqDYfGkX>ParC*C>K2w8 zT%7Y8i4ftFxZm3fJjahc@E4Y?AB155P*@^#$PT=<7|h%>XsE&+mx0UU#xnCyYc-KHbDSs=Q8^-&Bi2hIL` z71V6}gP>S~Q5M@=+$1p5A=1r3F}qke9g%+MWwMPZD>GRDSpR~&c7JY4qwYU{ll5;{zhH_4I_%?>r4SOVE1bE_=W*7g5eju&iY)}5c=XUpB@LD z>5hq2v2Z&G`pDk(CKPDc)D*1(x7y;lCf@$0w?AsMnDTDOS*2_E%jDM2>V&0U_ki{x z__2x9CHTs*kQ77!rcqqC}O_qWxNQe-N#IhB_C92J{ganM|C5E7(z6%avDPK7qT3EHnT zAnz`RM)bOibBvI0p{t5rluoF@<5`r(sz~uxjsfPCj{NY09k~kHhnC$NLyCF{s?;k< z&E}joVu6~nhTLPH2T_ydg8n5BY1F$1JaBg5iMt)uZSjl7utLf=g_yQvD= z+0bWRZB_|G`+Sjo-=dWNgNT#L<;0NW@4!kJ+M_GlEke`79^$*AcHeeWanSwFI@VoTz$DW-!a0C)M$*@kTEeMjTFM=752QqkuLDxE=10aeu!s0 zCkimbNIfHyUN?ev%2ON!c^Kc(LLt_rR~)j^ghU@ndcEK$hwRMu*yj=o^61 zq2$-mA>a0)+Js_zEdXo>Dx@kSyK+6Gx$R%m3|_t^74MKSZ=BHsL0|kaV)$aM&Ap{x zB4k)a4q0M?tN<-xyeXiHj;_C;C3SlDWo}ERPIgr#)5h#|wzZ_FsI8Qe?rXMTWCrPL z+w|X@aP$8lnM|1ZZVZ#KZr%_*Zs2~N(}2F%Np%kyq-(`Z41O-@&Blth3Csw; ze-(?D-Ty}{of_CMQw>Ms`B8q3px2`Jf_dxGa>3DnqKNM%TOwnbuH@Uj&IZH(9b#1y zv(DX}MUgo2HA6cMyr4T7F#NMvTx`(nS2J26&08HxppO`sgJw~)fjetf{=|XkRsUpb z_Jf((qKVmp2xo}P?ojHS-dZitK1(++;z!^?Aa=bd74l9KSzCuz2 zLwlNla{Tu5k=R^7wc~}9hH9H(W#B)FQh)ERj1W8Rk8~%tOzg**NpJZjuP5j=h=1ETr)Nn(OW=KN6?Q&xqhPMwnV_fIH`16H~2d9w){Z~9eSSB-s%c(At5$Ke?cDj@n zP*kh1jI(i{t{Y0_)&7}C9-tbGK<~1|F&Q&(qTQs zJ_ghU}kvSWw>=-J^|{vXtLekE~wz5cUd~-&YeKo*2fmrrmu@$2p=NBhcP4(C^%d*Y6mO0Z}?iN+vfzhVSAKlx&id zv2yTjr>@p+142YCw1h5z^H~v3;l={Ek{q6!&FM`qKNMn$wP2F`{}2VNxLsmw^Ig2T zzdi~aPGpm9==YtJpRWnQOi8H4Tp?c;W7T`oDAc#NHqT12!rect`^K~th`KiT_)4pC;&rMYr zEdlKRrtX^7|G;7_n6 zGtYfFf9AG=AHKSOP$>X_FL!{Hk%qlAv}$FKx81@oGj?{MctFUyv1?woSU(0!%Umoh zmh@vjST#4?1Sut49nIns*7$ZqSBCq|#l#>s0MoBPMSugv+RTi$r0E>|c)JlAh2Ti$ z>O7Rm-Io$f8{~2Vj>CEds}P5bR~_xHg$#rNsDldN|3dgK4dtUeFUC|$+P3*(Harn% zrO0-8cyJDs7q-vZ+<7jEu6HH`RG1Xwt5cCHP5>0#_P>GL1Y^yVTIicA9?ZylHJxDH zq=PJMLv*kDk`fHnaSUzQYd?Kr<%diF^;zvS1*EZf-2i|`rE2DD($tfigQHCoR#Q&T z^Fe9rIzWp{Hk0=`iD6@f=;b-fk1*+vUvop{lN2cmI$06xeGU_SuS zQnh7-F1ITZG=+?>3%stuuGAkku3qe~+2jK6H0viK%UA@__%M&@?Ijfdi{CeZXRHwp z%tGI~@bdR(n88R7T?l( zjzI|m>jykg9ewI^a-d%qCmlE6Xt38e`~!dAR_cf06b1TRZ#sYL7$O0(1m?6g6J+th zLcdyH7YGiZUo`|yRp?@vYw^l6Z3_!HrLE7LwDp@2!kp7J_D!clo}IUpTX0eG1P}$x z%N{xaNB9=_BLGL31mFloXBSxFh;;b;{{E(Lod3;L53SOzrH>b_7`0Zr$?=1Bt|;%P z^9CRnm8bpx0;}Xy(nJ)45E>EGi8!yp=?XB9>kS7bjiQXYvz3+XzPP(!DpRU8^wr+fyflR4z-&hf4H)D$3#mcW5PVNw^Urm#q=tuk zh|x8p6)KhryOg@st;amj14X{NF%8w~zMV$zcv@gU*>);3lexNuN8~?rBk`}|=9@MI zIMrv%_7z=j_pJfE%kO;I-1VoM6%i$+xEd$Fc1IQ$*=@|(lna{RB`X#z{%N==Ov!|8>xa^dIOb~*AG^l*3$cc{H!~gMZ+V>Y8O7|C7n^=YvXyzq zlLg|71CY?^$o3Lk-s5!k;z8S8eD;jaQZ$tbL1269am$05%hi-3_#-kw>l@&q!M2$p zuedzlVJ@_gQ;j@tE%84nw%L?tLkDA{Ul|b^zf36_>+64A?v6QJ&lxcM>Vicw=oKEM zDk!&bC7UM495bJ+A}V}&FpT>#b`IQs(c9NpjQ1KNKJ)p?u0NiphOb3|n(unf%PUCI zMs%w{M^3%6AvYTb9QqnFo6JKL3u$mcIY)42=4%71d_KR5IP-L#= zZ=hBMp2xd$iKW+61b*Fl7!NQ;?WU1_s42ABsp^u?v{7H>Y5wC%j=*Xi1wXGSjGEq9 zS)3pj-5W8m7Cl3e`atbYE*&lBrNl`pgkkU=A7V;r^a^R!o*4fVPQ3ox1S=piV6dHbV`e|raDvIdw~3|=+J8uQ_09E zzruKEW2XZ}75#ONLMHt=G?RJGmt~fm8Jf5KQ=jOfzfScOhHK>G^}x)lbSogv+9)c` zD9Fgd-@kvLZ8{ZLbl3<6n)+q6PJZhkn$oU8&;+GQwGm(C@(FGC)4unz^L0`l%g@ZB&1v4rITt_&^DVW3k%w@J$ ziUk~i^#Y(+$OS$(c+KLZg`MV6|GSUPR0QH9L7ImGTbv``#q%MOQZfSpi*2_)`T&k} zhrX{9|9`>H4D=3U!d+?~P;RzMWjN+jYl>oHVsh%hdiwfQ(AKI0#-a;8)f%_jBCKjp zM4wKif3GndSHVoRVD-oJ*;diw45g>1{|O3`d*@i@SWV|w0x-&kL_Sa#f@RF(orxSL!n{TpiiULf_X`%bp{-Z^RF4@+V+d{MeUQDha}jtftLYPIk$fdU z401qWCJxP!kYRuNJt*Ljz@ z)eY!aY;9$4s&4z=qn1xjPHR1kos@HISgZV~yyW~FBJv+GK>mqArlm&X{V-f+Mrn82 zAIhGVE$HmWjWkwI0S%$g`-T&(_n($8#f=;%YNw{kqTCTRN(jH8 zS-pk!P8~B+h*<#J7cbFDmOQqAkTlKcMB+J4CmFyTo547}!jXYT{&${+5T0rg=f}oA z8meBd1IB)_tasH&strTyTjnAiLtkIXdg5_rst89-v@s3RB;zeufb46ReOK_&eO($UTND$-Er1 z`|dS2TeMRytqPR<6*h83l2ee^61!)cjc-+&F`Rbe>PrE@%JCwv_+^s<*KGXkE-UNn zp0>0LdggnV3Y4G|>HJc6>_vU(N~_8-7zbgE&o#PkzES5$ms*VZiq1@8+5dswQ4;^n zMDP`PgjwvzsMH%Uq99ilJ@V{J3#UD;``W|39^`Qve`)adUnjuX@v$kWi#s5icyfsPd?*IoJT%xIZ++VrF=mKJ8(~ME6GCLp+yXDHJH>N- zh-ew}Xue}ZBA^FAgJnDiB8PoX$kDdbOO;nGZ-IAOk=pA&4c;>zo4x?ZlG4%;{_;KA z3^ZA;XT{3*>fov>zK0mo7pf0~hT40QZY=Y(kT`e=D(lMS@vrilwWltM9p&`CJ6v`5 zdnl%>%B5nIN>%H>JX1olcQ7`FSwx~;E9V(5SkAMn1exw1ehIw529!Cv5M!7;%*>sw zM_3ijxuXTzM+Zc^_iBRo%|oupw2ScpN}U5q5S_t|Rqrlu`014qo=*G%3os<@Dg6rR zVb{Q>2TC|&E=Ul*dmD8sH!dZxW+swqs6)i_VM|LKFEh4TNzb~ds(Nrjg7G0pDR_7< zb=+HM19#YIXiNc1#bF$7v1g$qQs=3+mX1tkK*IdFoUJX(H2@P$ zp1hULLgd zuU|wMG%DWKqdbbn{l8x|V^8|iV;MN@XF zF3(ef|LWM8$KNH)iO*kw)UddhP?XIx+#ml?#&E*3DUY9J-UcmML&p0S4e!-VfLH25 z1|`zW%bL@OE{eCqbw^%_Vn(33rp{=E++RLiGd3^fDQi(j!kSzf!CzbOXo`Bwj7eLF zp(?qyArhxK%>!d2oeWh6^{@zkjFR<_3;}WlnrJyS1rIuk*`5T!yLASme&Un&n)eQI zEDX?RpIUs!O!IT>+NlZ)%C@ACk8Y^C+ZT=HrV_daOu7vI$wk&Ar*+=Rl)@h6tiuRS zgFRcsBGC~W07?SePO+e%&|G`K@r-%_pVyw_4N}@C^JQt+l_X25;6LNwICtbQRelNc z_=0`wXVRvmZT5hZmQew(Y1ZF<{&YuDq;0DVPGSA}0TU6)r!X*(8`1_z^H|KFhDUCC zgSF*8VK3xQ^z8xZYd83RTu9_mqpZpt`-!Zj)us?E9BeGfdsb%t{I0tZ;~lF*B|Ju} z$h{3`JY;!l5u;&gC!2VDjd`4AmWJ*1Lm%%p*|1d}e04AAwxwp*B@j?qTBA}JM=-ey zv3V@3%@}*0g{B?u4BaMh_Ho=O`1Eb}Gd{%}VjUh!n4*%Wt?1TloLW$oCp>9JJ$7Gc zBcXfaE6})s15cg_ZyLGR&RnyAqzjx=b6`I)xsytrnt^+!LKYt zd7W+EwzePLN3|sQww)@`Wu|Ziric~JYn<+GZYZ zx!(ZHENcM}UI#P;wypHC3b*(Q58by$1P&w2sU7w4bXcE4z7l^7@F|%`&Ckyll5iuu z){oZK!OjjHq+kk+o%#7M%p`(l-s5hhv29&{8JmJ}?Ns|H`l)PzxcBOeF^+?8Z?;?~ zS}j{eeRu6J!tG|seq&680CEGz-|U9P{0e8mod7DuTsdscCYE7ilC1jLb16-}5Fv3n zUi04sBRKdAAEwxs=CN0cIDJT=eo!t4v|6>MH4o1z2=$r6z%f<6I-KnT4F7M>nCYsg zA3xkeg+Gnes4^EL53xH60CK?|D)7DS?QonwADFOUzmd#BHy5E~G|b5lcvcKsFhj35 zQ)Mf2kgTYz1gWiwhDY0}ml+yf(z=<*A(W!H3s#2Y|ZmL{~3jjavY1^?keyEJa z`$lW+et{l_O|x&UiuUi5tuSE4l=0ZltAKOCmEAYr|MD`jMkFsV z#(4-!_6`DX-5NMyWOX{^5&Z_kb1s37 zA|&9sBi6}4?ofB%aiGNh`JF`f>a5YJ%@r%+)`+86^?n|wcY6`uCzuZu)Pq9bj9B5p z71A={e&+=;G>M*--wqG3e)F%uesJ(3>?68)10D^qUaoJtFzx zh(A?Jid6pjlQLJVIEqJqVhJ%dVeJvQJrchQ3RA345QQau3ib}Nu$7wE-?#6;MaL~});3>5CI`{gDZ_L!6jG_-F0Jlw!igb( zY&mHyKG+EHcf(bdNTIKUqU!4%Vw|-2PTaXaoXP|JFQ<7B`6j;JySEa+Mwh*{PQTr; zOqddlW~u{2eSKfcjUMggCdaa>s;H|i;gB~XZB-wE$~N>joYr}!WKyitQICVDM59c9o$cKatwI$-BzioRfV-d@q z_W8GZCjHe6REt6gT@M43Aufs+`u5O>>~VLfEjs*qD<5)|2LFYuz-L*^+FO}=5?BRB zZ#+pMJ>^s?oe6DsH1A_PRbW(o;RoF1ZzIC6=;+|MS$6J#Ki%=F>FjQM$h5xP0zX~A z9-hWQ5u!yg+xoC2hd$dZ@5h?#zGBN?cr+F=nLe*3u^I&weI|tzVEX){G13l zS~W8F2;~f$Gt)3LjZtGI{2;M3>u23R*nil~ympfoeUfOnxs>Q4xBg+yJ0V{U%?DMV zn3FMF2~~y z9uk7%>4%q;X1~l>L5r(r)GpGTy#BZvnSk%&{lbdGP}3bNkM5TzN{C&=B1kfIpSU_AGseO&dPLsLor z73BdOsfy|UaiplIHkpp10v^m^FLm1j3DJ!`aGWhw(5EE*ktF}UiCmN5?=#fq%S~9p zIBe6#J189QaNkbwukKZ{Kep_!+u^r{z-W*4kz@|7XM-p)p1QdbL!y)MO#fC1zY)GP z_20pk+nE4R{baHS;)q1Y0M-X+l|{wj&KQD|6vJZb#2u7Y6;DQTHOL;`W)wpvv|&H`F( zL=VWs`>sEd{I$R}lEPCs25r+!w1j5b6NJ@qFN8#_$o>x3Lq>!@&IvE}&r)ycul^kO zGiqK@FAAKB-!@%y%RSgM2Qb=~AiO@O9dH>LlgzC=`#a#98JQcK6)xv#)-lz_ zQNPPIJM5jy%GL$jaXQ++Re5MmW+QDpz4;PZj7S$JO56rl`6noGc1Rgy^s+a&V7lvKrBghXcK}w zwS)qBS%M&4k5SP zI!4vjDeVCFgdmH#+Ej}^DJ`{6!+UY^`8_A&5VE%=n%Z^wd!ZQ=Tzt}pSD^UBA69~W;nEI33yg7hu^a&{uGX@>=JJkj`*}4dz+)VmFXy6fRRLxlg`1e ztpFVl2s8+BO`Ex+3G{)DXxxX&qJw%RLRne)7Fd_)Lpf_?WRxeD{}~yd8@54^we_ED zzJ28;!nU3{>Mhxr2}?=$jByMpkgyF0;ZHu+~ z@di-BcR#S?>6STyP|x_j5`&~~=_e;mJ|@Uxf5j!27cT4ygk->40Q%~?2I#9UAT$O> zZ@+Tbi}lu%wad;$9XVgto9{DDDZY6U_5DnooY;{0s&+tZQ0qhXJ+y4%gCH5?SI6GI zl}H$OJ%O**+H2a9F1Ykvpm11TF>x|T1H07yi|QA+KjK?#V9Uaj?e*Ty;s!=d{pQQcbh+q zr~oMp+ZF!}v_`h<8Y7x55N0}<4}ql`>29tqgyLAnoEno>LHu@htk1lzcua1~{=J_p zg-Y_H{#Ib^8b{ujbAkdL7nD?2eYS1sA-932*ME<^-!VWOo2@(4cvWIhx6j8y^wNR* ziuF+_c%%2Y*M@}a7Uh1~Lrw>smP7Y~ZMK_Jy6eqBO#Gf)BB>2a(f8tXA=7-#ZnEZsXgSWC}#gI{}QGw zy43Y<;2+I*HztW+QtqkKLYwI5Vapa&ZiQ8ryA_%0>kCf@D8o*yP+;vZ)Y7#8t3f=_ zq!aVitCHz7Tc{5OBtnnD;^I)Gqg6+9H8G%CT#2IKXd}0xkpVDk4fVK9k+^1s8sc=% zK*O}hH5{jWM1=0VrqkuF2e1wWvAw;WOft#nrS9S@S|prui^W6v$dk}Kto9$7B#f&nVGfm9GZ$qy_2~ij)Y0D7fnD+Ue-O z*jRAqkOR$!UiAsF`DM!+Q&&O|U6;T$yRJ-C9c)ZluIp95t91;}L zE&{Np?RJN)D`@^87csPb^?u3G*-+W6_Lwj(oJ9yuH6}zZO=0(PXuv7Y$Hrq%@JSQh zXrxQE%?_sshFzf*cU#nXRi-bE+uL}UcW9@&^tiseM-{POGI?YAVF}4=?=L^mzGRT(`a_O-HO%xTW6^=-7L37 z;QsiD!0k4jJ~2%t8QQ`2)^M_5Md|C{2<bg^w6Y1KOd=~^g)=Ar0U9RmQ3iGyM%)b01U<+KWP5YO;Ep8j2IJne`nJm;ED1vE+mLj@mq?dge2* z)L~<~Sk-W%9>zB~-n)S1KtMjoXv2!VoK+#`%D`4Zp`v_NbfTQqwcmGvEsI#}NPG#DLeZU-^?W>{p$lH{I?#DJUF8!PW%8L#+h*^`BQ z$k(%}V|x(J+6Uq^QNT{1O`Cj8Po}+W&0{YzcU(=Wea+(JgNlkeR;tN&xY9~+l~~>5 zr#U33HI;z9r6u|XBulPfpcjlt;wF>WVxztx_LRk`iVx^jE{XZdDIY-WimL5&y*-?G z2K1RJNEz~YfHE1eNJv*tFphMwgy7G(GrWSxPuuGn|B**IU?|b1rX1OON{KDH^{+hUyI)A<)W0)O{BWx^rm*Pxll3mce3P^(tFC-*?>{l3 z(ot5~80P}4>cXCdqs(XK1P~$*mY64p(jVZAjrU7_9U8?TA|+P2eJa|RMk;n)@vg2G z-$VN&LQ@MwE;X~n8dLco)a`Mj(tz)Ad%IRZr|6905>D9lk<j;gQ`=X4I8@~fxbikKwirG;Mk*x|Cis)2~b_~Br>ctTMgoFs`oM{?X~ z3sY`9TXdaokDw-}LGvF4jw+Wxx-BkKH- zo>)Nr!Hmfv&G!$b<&MKLHV4y4jrJ!B;9pV?K+ZdsD@i~?LZ-depsw5O)SsgPtbBWQ z3}OFq48;GpV^E9W{2SZ`>?Guwrhdk@-JkqqS3IMr4o(=I3KMZK8ByU52LCl|jo1Wr z2M>$=Hx(I99FR%$sfQgZ2nHFKA1GsLiRSDUFS#R2 zjFgZDko%#&M%Vuw7egT-1o)|#SASbiVw-mN*ldm3lERIxu-QlHG#N{&cIi+L7s$Fk z`a%o4LnC&ey2v=NmHFSONR!E`sz<=)^72h{vJ|T*_LGg#PuVFBZPuw6N|kzcZCY%N z#m3uLU3EEh0P_$5$v%ENwSWTP+8SwvwL0r5(shdqXE&BYJcKB+-PC4E)B^!@Mtdt* z{;^`GYWn9KCRib71n{uqL7S#BE9?9Jyw8M^OmZ-tfzFdnINM7FF5gf}K!JQA5lNPA z@9FMm$&noZry>9>y<6K{58#tr47Z-@C;NBk1ct)OiilvEC}up9iCAh(uj+Cu(Sy^Q zlHoX(HOSge^9JW;O10b)@6~ELMPd-s4Yi$-YoMxdz01GX0boXY)I-xp^$l?kZkn?T zbE1(^s{f*66=5jdn#H93kAXjEytk7UN1i)R!r@q`E=bmQhT0SPP~<~76By_XBtW7d zabci|h4P(10Y-636*l_e!dPAN9r@6`03%y!hlWdjbai>fWK5#$vt+(`OZM((@}Yzy z{!hARz;4!lbUY)>Ai?!@XgmpXk_6b!Cxw0iuzJqqLULc)w9RS$|=|P z5OGJ>Z@S?lB3>&1HrL;V@sx)-)ffdPE6`sv!zm3R8XEE;A)%kkt!0>+2?6sJ6%`f_ z`%v6)u5Tm>G?@+O!F^Jc$-H)Nb36e1C+R~&#c&28>piHJ?XjEIFC%?$)u;CW1HFY>SVkzKfKmJ61 zz*wPx{Ytj%|Har_M#Z^p+oB1s0YZYihTsz1f;$9vg1fs1C_;eX9^Bm}!9(Hhgu>n3 zo%dDN-e;e4-?^{7f3&t%s%A|YW6VB!@3ZC+CaBu%ny&m5lH~b+?%*Ou0mirC<_aB4 z<@=wS4cUF#1>M&f^Ih(!h5+F!98iwt3sR@r0WW5n@Nx5;(?Mci{X7|zo^3Szu*YE) zS(#-2r8PIsfoPUlXh81_cIHBN^V5 zu4upR@6Kwdn|c$#atx_aARChPTc+R0iC4%3X=Pd`@4kVoWxO5y!(zQF>>1T|Zv^A^ zkRhAgu5}u6@`;>W5GNcd#?A5kci(B|?~$dC=`ZK~HB)%2cAw~^;;3OK-NQcpcT zH8y;1MB{151?3A9O}jahJ`enQV^tC_bcqi?8`?2Z4N}#LUp{ z8Vh|8a8Y2BFomqo9-3>qQRxqRu~L1QFz>OfDTLD?YGA-$>ggZd_a4eZys`)8H#E*4@ijNM2{6*gi{MGv1%V zG(-2B;_+4BiJ(>JZ;~nSVaup zJ3%Z$kqY3t!)rrZ!>aK1e_Eg3V!blER*G1ag8mttE7 zediLE4B0ifL`f`t*on2Myaj7Zn$VsiVJNbE+c@IrQb4J{TYI}WH;n~%sk{!$tm)}- z;Z#(U)13Fc+AHwy7~|mD((-;9UKgBTZq$!+h+06vAFqd8h$naEuAiLuXGsch zah$jVl^C^WsQS}AS>pM`<-CXQmi;%y8U7_1oyKmCrUlzF23AW84Ya*EB2^qmlgSCZ zi{+8rBv#*BPPSZ{b~GCw^@3gC;~_2ie(<)!NjUW-D06Yt%#D>}A*iRjH|`3Ys`;$XABS#c#i zfe&@IE}{903OR-@kZnr(G3uq5=VuKdGYSgi|3Dkoa-2TN}cC;_=F8f&{5uW9iCu zoN)#F?EXmeKd29=VCCOv)cZk_uVh0MB#|>)MLPXB#DQf>i_5TyChKIjNo`2ZG(5Cd zN5iEa5|N9L2EVroBy%B22Hu3k&FyKp>cIq;1c8x#p}55Lvfsix;v}yf@8_t+_>qGV zA+1g7k3HbVu}B4`oYK3DjEt|CYx^4qD48$3?CMcsR#jIuIs0bX>W@Z}%C4!l9t$y??{Vz#YqRXMJTyDE zL=X}rm)P<;OUBNm^WC9JJ-T#qEgQRI+<}DM@e17XfLgTt-0fWV_V1MqB?b@7*#Hx7M9k2Sn7H*cV{v32${Uce z0V}&I%d>+&R{Q)D(Hh_zGd9dP;W|v5{f@mDTouq>ZGB0+YGGeGLd(Ya()yW~yiB9O z)YY*L0K)5VB{FFl=DxueR$6cLtn~~`g#uxZ5;in*O5$=r@JN&G>Frc0e%`QdMTR4E zqs09Oh$GBS_%Gmepr&`%yIRxdBvdJw3>Vc0@>3$Fl#s?Q~$+)@irKH31a>nE~4DZ4b;h>_OrBNIO4{-JesXX^Yd8rh@~ z#=)$Z8;3q+m%O}mKx!eE^l>nkP+H`cEnjim1g;sKVFsr=zMmd1Y0_vK=7dh1`~ZK8 z50a%_B&&)U7&5H4p0AUx#R?yUZRQxpSX^2dM7`&!lkIHle&fyiCfUzg3j7d%KqTO3 zB2;%YMJ&4fh~(W+);m*Ak4cRGO3_}(=W<;ePOXwLORE2}ycGAe_u23FBB2HrF_yxe zLSR)R&f}Ml^A3IU>#G}k1tbA^$SQpyk%#T&J=u%?fp&dwvES0=<)%6|Zr1~^vS1@S zGD$h@cx|B5cLEqwE_cWuz3GnZ6DHv{}yuY zNNwU14UrnnrLn7eq8qs_r|4@l0)za4mycEI^YXz=t^%I`#fCP+M@8U;-{eNZlVb=W z{K4FS-f!taSaU=|nho*f>;HDsf@LH%iVvmIBmFA5;X`P+2Mrxz3z+YS$xX^9p5)XD5cwDEb60Cg_i5ZEoNvv-3iTm^&7ls>2XG)@CjPyCc z3YH+Vau0-S#oO5lk!U4RGyw4gV{X_NM2fz{cMPY<3yCCZ_Ew`1>(YDoih8Iv=~1Dt z>}M9~y*q|k9Hn#7Qrr9>2+#FRhLE2JX?wZmx3PP2ki+Hg8|JOzNu`i|+|kbmzbh>6 zMVLL*&U!oA8?rcgo(iQ)ulgt`4Rw%M!znh>_4hH-G)SLC4U)CM2XbzF6(UkP+&poO zDp7tX=w*T7v{7qQtFSPu$4mBjJqqc_glyC5Ll99aFIW4LDSp#l7i@+KaSf!zT>&S2~I*Zy69zJIruT3WYHvg@Ai4A?z#{0 zHM7g!<@+CXJ{f{EPuEp`w=icfH%Ky$=Lpw@QYFn^c_4A#QGTGtw@zaEZBLI2PVpJr zU9x}oAceo4AwZcr{TPYH7DO)aA>m?`MGMSe=5cjHZg_owzhSKOR^(TH*LNUn$T2U~ zL(t@tyJx}6h_ysh*Y3hU2^xwHR`aSoKfi{M-)|PvJ8XaIShuW8&>7M=dYX~7Wpx#E zzdXj#GiWv1_t*TjO8QcilQF_j;R(kquKB`fy_W6o_8k3tNj0C zS5-DF_Z;plu*>(ON&n*0v{db#nLuN5b9w*4_fuYFJ$$8{li=Potu#FPTt>9bPM>k= zXIzd;F3B%H>1VdnI}Z%<-}^zFwRPS@a22n_;|C5344kZ!MeTHHCL1uatTO|^-^d=h z$@Dcw(!0K6NWNYp{Ce!SNG`Yf)|SGsfR|X_&wFj#qfM51HJOz6@|u%}_WHh(2DQs; z;-GuyX~0@US(`JFB^7&ZCPxn!jrbC)B#v&uCCSoD;k@n8|8vL1DU4Rm_923Ld9hzH z?x4~FAN7#V)56#H9|%7$ZDXI+(zSSgNY!7v-rtm?+9VDgY!LP9X?$JJ1JTS9&!S8T zw2s{KJDP3x&vbZI8y8->G-`;0TpONBtE>_?p6~Ae{b2=rbzNGKb&^%oGJUW zJuabM1bz3=<93CsrD>!5uFibxOKLf2mLrWf{NyrZEn?uYpF29{z2Z0D;PFLH@bGoK@OYBM7!q5poPmYB7zy=>CV~~ zl)yuPIGd{yO%3C-0K%{im6qP&^h3@N>y}owoT>csX{+N7-#(if@>ZvX*{_zVI0NbkI0%a3TBiR6?2v(rU_Nd6t<2JZqB5zY+8`>Dz*7rng& z#iiaDSUy(UC&itK+RP>y217sdHQb%+yLU=w;>(`BeKGvX8dX;5$R)h^YAPx6qT<<--0yZ(2aQ-DCi=+oqS|3II-+RDOttR0 z%2V0+pw=S$ndsS_OYqqbeJ(D>>~yZq4Bzj;OZ#YtJ*9O;Myl0g`~gfVKt{O~=diUS zL;&ImEGx}mcP(>}(_T-sm40?06MQxT@VfEK%?^+CM>>$y_7!_I^SBSBrn&}CrNz4T z6S&3n7;9>e4Xw@ZDW~)xy{Dm;jgnpFOsLA##Eg=c2i7u$hv23LUwYk}PdhvNCy%y< zGx;Dd!MR*#ast zIoktjbL_{hmc_S^-Y%HaPTqev0~D~a?oHcyj>c|Ed;mBL&ox<2$1l`2xxx~q;V-v9 z5G3?b;C9;=&vM+JTHyBZE1YysgXtGm>|6k<*GDqiRJ8fcjUzz@6AOtbzJAh|)3TVc z{pS8l!X6?_hBWr>E}ya`W-dM*aft2ZT4EFLBC+54$1vZC=s~7Y6qyft&+v~sn6pr3 z4g~W*kX{AHCKR)ms%Q}zE;XAHFQxO>N;R8IUR=n?eCwrPlfPbW=Ujgr-CoTPpagAS zAW{zo-d8PaQ2oBbR%CkAdg9$i+d^}}X(vW*fZ z>vAy?G)8FoqGJMg?(4KA@4d1zg)~|g z&bjckGPQ5b&IqZvav*p%G?C=x9f%Cl(`Kx}_z$07n9?SKGd#(xepIiHmqSNP;Jm=! zgEu^A66T_4M#zc1xppPP+tS_*iOf2jghdx#ge23pPKH-V>x)Sp`+NU11P>keGX#Ol3Z350LzWy3O$XI96(xLttyhB z_<|F@+qhGZI`g{h!+Q7Am>FdlL7Zx5(yNsHWs=>ActgBem&i#GILtG@-Qw^3fJ+#6n{J~hDTI(J%6{3iIuLV9 z^ZPUV^%Ok48mD^er)4saGdc8dVL2nJtA)(aqIf>Azu4b{+(+aY+i_{FqRgPMobnSQc6|Vrf}mV?ie&##99WzHoFUhUMkdxMpYD zWqe;C1jHW2ri*btgrWHdvWOV%DD+?Cbh$k@p#O4PmYXb;iN)MlqA0;*mC^K;(UjLTl6osTbT-m)3GFqE?|?WgZ9) zZBNxeiv214*28j)5bV5*k*`;?d*^dU9&X1%L#&3gPYaC*&_+BoKM^(@ew^lH&Q&X& zfkvPwWa+^^1bkI>gI%#u#bJukn(4zV&MoAL8xVbGIKYV6v6YzNbF9{FR$WA^xftA5B$Q9|Hu3DKkk+7El3z!k6{=}y`<2f)$9scu>s$S_m!f(0<% z8NR~u+tw?C9pgK(R!By*Y)PvUig7-#^(j>(O`g7cZ?LVL4Q(mBJ<uM*ia9cXxB{#p0i}z?p^5J zpw(0HIctaIuCXtyLb&sj>L5-lW|vJi?(Z?`)VEPL0+%s~W&Dn1RWC3DGFT(O7%=(H zg^FJya%HpINpmVa3Rz#rPWGjY-+t%eU_zg-%qL@G)uRRL_V=SO!rF|FjS?iZ>P5@A z^&g!r0ha};FvBVLM&9CdDlkV4=QV{^a&&$(gs4 z;Ut9OQPoW@cbX}*L$x$5@|@C?*xUVc4{Q3@nPP?jITG~V=5AAA%|O}ZN8j6{^_?fwa__#G3?ny$n%GGoAzuEp_X@f zYhU;M8-Qeh8Z6KXAwif2)#iBRPtZmdc9~gg3ELC~A-GZ2VU*G%di8y$%MLaVXSA@lM7l+OHi%pX# zlMW%hi@lm6Ug(nl9U=Ch6ut1!)YhDe$;U8yT1e~2Deoy9V{Gdq-sBrBEV^tHp< z2)z5KZv|7F@?L}97Az~P~w$^gY~zFT5nmD5436q;`~{!WbM z#ZF}pde_TJNYIc7Mtsy<8dsZ! z;g?S7BXD%$#}UZ9O*ObeYC|u%T}`ciTrshSRn(9)iE)KwQxUh?@gw!#pZnvQ+yyZb z(w7pS!SN8fo}N2(+;lL5ho?g^oxyhDj+@bq(0GP?v=*7(5C91nW`2~PY}=5{6f|cSUB3afo;(5q`8GJ6p-H zsrz!NaNC6_BoMQk765NT(&bZrsdTqin9(x(%u(&W&!=f@T+S|u5xd(!iZoe&8*gJ< zo_H@uHNQ-33ju{`CU?Ag0zkRQ7jWO&Q#Eq^SK0f3Qs0S=`Xs>%*z0<}29* zZ{6@y2GWHN*;C{MmVVtK8uD^Qe9}iczq~iR*;UP{`kL~3lgyp44$Vp2*ueQ0mU4-U z4Z&jCabdX$a(9z2Mb}>Qti&E7FAAYqYtWW!Nl`26AZ=XN5&N#BZaXf`P1|%KW6#Cl zBFj+E+-vD#OOqzABaU6;q~p&E3(Iv|TAEK&S|bvN^)S9C>s>F4){aHh{P14@Vet~x z%1DOjsp)6i$AMQAGmRmKP?{N>0)NDL?If&R`FBYeMcyc!z^{HH4fQp)Wg-C0@<*L_nVKQk>yWJdk|_^Nmi)Uk`Tw+`mYwg# z{7CygZzB53zkg~&qtA1GwBGw;5fZfn6x6W6`%`8s*&?`Q4ELumXuI1`{8KxR4C>eL0bFN*G#H{~*`<5U+kVFFlROZfU8&ahR zI=j}1cJ4hTb6<#0A(w!r`67)DGZyMALrwNPsA^g9bpWE;ZBulgfQcA8aY ze1TRoSQmvGc*Sqe+rh=1_rZGM!66$V8r1$Sl-pa{+FEfjdzCCtgYX2~ER(r`J3RfI zCc}F<_03AHc25L+r`<^c>Q*EIA&`D8GNyxS<>5Iua9SoOiea~l5q_6({yVM0ZI=CW zO*-ESLq8&XMKf#*1v;GlEEg^+!&j}OB4(I%m=eM5`fVR)$0DdjbL0l4xIA)IYOn3| zqI$27QK&Q+_=s;^&d&mxL%5bw1rc5QK7C%h+*lPTqhiNbAQt=*Pvyn0!_f)T^R8hCE+12VQgyZJS>T7Bb|yB<8%Vr}fqb}k~O zBa)l17g&f+JvDVv-z<010WwQ$c@=-x_8rC0C)U$%QbS_s1~nP!2c}AC`%k&F9_~w| zzb;usC>!hA2qfV1y-b8ofw$Fw!OEL~45C zy_>LR!BGg{(N6eIq!d9%YIocEItD0H$fzGOBu(nmZeZQQl^4Q+|L5`>AiME`i!)LC z!jig;WoE9BTu+{k?fjYtx4*}EZF2&~EJ8hupnVXaoYkUnFPQK{mz-qAhDsq>iz7@D zX$LakZ>t@0GiyO!OU4E>o9%57;4_8@F*zisp6PuCulvD(o&;86uSes zYOb$$-)&1P$l0s=wYVh6c>YceG~|)vT&b~out)P_s*Czy-1k8;lCY-!CvM+s>g9|4 z43;830V8&8eb>%&zabNSdB4~ca_m5U1+2x!hVaLpxAdnMAYvIkq^c) zzz|6r?zbCX6Bp_di+ytbevvtL?`}a$O{u4T)c^(lPYF)nS{C$`S`{>T?}VZ?h% za(RLL>R{o!RLvk~>rVVJ6LYJKM9DtIkyf2UWNmp6tUeWC?6^<e$oe4^=(8AnIpCtov!(^Jb76HF+ml7v zCZs?R$WRliQwj(UDeb3p{^6k0FKyb)n5g{Pj6=-UQ}H^$Z$}FTmi0~?Ak#zBsy6}; zuEZ9tlO0TL&-37)M%8!FV~>K)g@+9U`iM|BKL&;5^XjbEP)ei>r%YHk7C)-U*Ik80 zAktJ0Q&^{1wz*Jg%vW?za0Ks8)Bv>eh=k%ORb~d_`xb5|aq=N$8%>|T^t*Y3qKHF` zGSd1`+(V(14LN(SPa?L4+4FP0dy40I&90NBRZnzy{ZZgG6+WPjp(d;kPvRi)&dPeyO&_b6k529P$Fo=c;TW@>Kw2R}+an?EH75gn5nYg5zSHIt9s zmuEI46K8MCYRM$E%hLtX7zw7LX;zyuumBQ5`j=zX3Rtvt((UxmfGGgE2OJ? zY3(5CcC0}+J04`NXy?RRmMtZ0vtg%eZo2so>^{2y$&$Z^KkyZFGr&qAH~``rY}LOg zDK3zP&0hWZf03QT0h&;U-|w}-fj5f4zM`A?AG}%UPUHDYnVV#@zrRKkeDDizx|tg& z=fwFKJs48Y!#8Vm4ZdDAF&A|k)sLvQu)c{7G(4R5R1qOmJSJR(o_@9q;+X7^jfq0up z0lrlJ2%vz<0j(ktEYDZ-_TYFMFQ&tiLrKtZR9;anb6J@jRHLwY7Cm>7zYoybX{RA) zeGp9{>XfbX?WP1xHfxj;_42U!pZ4?si`y5Vqyrz(R6X-GNCTwTEBjjM6)m(&Kmtq< zKmslO?F0w4Wp()G?WTdVYo;3}A%;Wab_> zDkQEIUBagk;U}p;Ff)()(?EJ}7N1Qx82A{;BYhq1+@$Cye^Bzfto!PJo?}2wl5DZv zTOvg{)a-T$wytbNu;xQ#cyPt#iED5Y#Ps|d)24<1;BHuKgyxd5f(goOWs9JmhTK3m zwzPt*JhdeLa4+v!rCO;vx7$Io^6Q|vC=!9NBz>$x*J$hO z^Q3Oi)^tGTy4~~=5MXLOXE}k(_zHNm=!6&G&juAaY1CpS@rDkwoXYaO5#G)MjK$}es zTXz5usKR2>FaqKSEWmuChs=Uy{v(e_%6lapifqCOR$cY6YT}t6N$0y% zOL)B<7oGe~G10bn#MmS^&1U%1;+h!~KpmH~Diw*$BDlAbbb6D3Xw;8CZ0(-;yTkb> z$K7lB(E?*rPWxigX}$D=lf_?2;}xC)#q@vBLC|Ojo0vfq9h>p3-EK%+YzF>PGA{{c zs}r&mhb%T)p@Gc%xU*@o4W9kghc$){oK4rxVljOPBB4bd*S>?a=r+o4WcFkDuM#VO za&-p~Rw-og2SSlz5L{Nn9(S;3e>eL3|7(_@a{c9d$R>y{AjMtkvo}?oEa2VH zR-(M-V?i91-X0k$WGCvO+Hx1%LWM8uXna0lmYX-T2zl90XSt`>oc}KcPU75MLurVX z@m)vHoc7*;E9_qsGQjtS3hocZ(?@MH8InV4-xM?Bj7Xb#Jh6fWaWD0pyX(o$CIW_` z6v6b-q*YrLLffFPxUHD{;=z)C>8-$|6$~kd|Jw`TU%Kt1Pi$~IgA|K&-hj!S33?U^ z){~Xf8AH+{F6re84DPYlj94xYU;c7E zq=4$3+n=8Z?dA zpMNwc_<8@)poFW0*i0C_#$ID_UgP8TzA%_7QA2fU^#s(fZ6!HLK0Df5_I{9XS8s)q z(T_~hJ<1ESE&rmmLt2W5=76_Ut55fTeyR~9J>Do^*)M;F?gyc zQ4Xj*62$)M5>k5N_H6JsfYYfuGBhOYBF1f}%M-F<$_fcTiQ?TORazY%Wu$KPTFNA|xJEAL$U z28Oz7iwZY#2dEz!-Ka-0x#2W4G#GS$VFn>$&=4x_F5%pj)G7l*SMoOq^Vii4(yQQ6QO+8Wa;UEs z0|hX3_2FJwDzYwT&zNovZboc=+eka#fvMvaRr8UA;J`!4>UD(*C&SBD0wpzDiA zD=f^X00`m>jgIi66q(@8PS{M}EBqy&bNN|F%1Ah%j8Qzf+pk;)wjwO`*z`V!@a76Z z-lE=T^*6Eq$S1<^|7Ok(9ZHO=MnQmLKidKYy-0!DfkxnvIR(yp&fsXNC9LnH*MXwl z9r!?;f+H1(YNd`lsnL*osv`i8i;2H6SgJS8`M0j}qb=u5BHltiH4WM>lDD@`G3p;4 z7@AOF2{Yb?#XB@bl?s&2xGQJiEA|*Jyiv{T-25@>AGDLZBz5E9QgCIU5<1h>@u(P} zaY#gGxa2|i>F`E@keS~gZn5s8?HY_^mKc&r+@cw&kl**)^X+xOE?a^EGAr4#v%~fn zMc=BaRU&IE64(O3wKJ7e|ENScGE~h6$pC@LhKbkrH;Wskfb_Sjx;n=BLsLY9wTB0H z+w+sR;RPQjXM?6`^gkK1>z|C7k{KC^vUR00Q-Q%FB+;<|WnDnaElZ{{Miv%K=j{=3 zAn(fUt%ubOsTTTkJ@I?s*Zw&WcF~RH#EtWpXh(Fj(M4VU?unmqG(uybuP+?=BK-HE zd2iM85s_iM)}4GL038<;dYNu-?QC4XBs#qPYQRvSJ3|bO@4Y3n8Ni7X?Ue*%)@vkp zk}DJWvfNG^NPuewN)sKPFWnZX(Slni)>)VEgB1wyjzHcfo}Zt`Weie2X>(lv$}%D6 zK2|_E>hf4$fh&8HX9*oY^cq4h|VD++)2aDAymzR(%BwXutBG18QSx8M$Oj@cFE&!oi(!iPg&H_SX>BJ6kx5=QdnN z{AbXxxVHL};2WEEb5q>X0gw6~A5ERqnGAyb2+Ak}7wCw1&BPS|i7=bVe;?8BufDV_ zuDfm@klY#)Z!Rt?@W$%jF5jv}ggFM!Uz4m3W?rw&PB-%&cJ{Ol$;y`e9Zk-`?%Fpl zFOKe1X7|AS{9ukZK&&vudD--*uZ<7Yf%Yl7>6KA?A%+-4*ED!0!}!y3J1l+#)8jh& zbnvD~iWoGba}fR?i^T+k94hVMJ;TiHx*kISBC1hwEA{HQIePRvxpGkv+75KY^0=Ua z5qk23yI0ijti3UTtgH>TaMJ#b_*|6ds)W-;-?_XSgbQ(8`}%&=Hz+;eOJ>PYo^U_W zxPj!v)fK)fVO+Tj#|zk6Wz`;=ur-5A<^HHG2amR$kqgg!p$xHcONm2P3))oDI?N>OSbv${neXVt0Z2%0tAqT2I4Q&buCziOd zh<6Rs0LWJv%=a^ZBo!cn7@M>I2qfF8g%h_8%SDWGRe0*Lj&Bq^OW}!nkx`P z*M^ci*(@hof4SRktQDkWG?-DNbymlIoCoi#0(BDg>1Eh@dD}-pA7zWIF1b`^9@>dz$^rYK z*CJk3z^tQDkOJzHK1>q8M*~JJgH}@5A89PT`k6d;kd%2b0FQ0)Ckq>gp_KXe!(}c1 z08a&K5i-ED={ddiYpc14!_yWeNU}F5*3ykijzCz0@Ur%2wM4|gl1EzxslLr3Qhgw7 zAFmtm=!4~!E*CPK6N$aB!T_wRmTDVpb`7fPzU~-tz|_@?+}Us5aYHALJeU?(#=WNqGUs@ zdbxDGvV3jZj>OQv1;8mJqr)+bCQ@e6!o|kMvHGpjJow&yWq_QWJN1ls8uv#+UU`KL z0d}4+@!ZR(5x+yNqa3%keAAj;Nm(6#Ozw4a1I=~2rMDZ~0R<)zP9Z(hO0;9Bg8D~3 z@7s_*VX#i{bmHa=N=SHH)-U8J zR$jN)wT=TFPk3aCZdh3v15ryO0at7TrDQ@gsjTs6fCVHJnTp}($+znslgtZm;Z5VaO`7Aj3 z({H|60(&si3ty_Ol(Y1QeZWB55AE$hHz3$THPlunz#p7!E1s)B@FiaGRE_+?grwWz zNGjT<84+I>C|PJBYj15EfN}S;^JM7|ge7Gw2t=$eluBN}MzX;>cy2t0t>`@xhpkLH zO&e9x)Y8VMQR3rvZeEy0`|+ld~VqQ6QgiI<4wWHD6a`4Y~F}>mNSTk@|WY7NIT5==82IF%X1o`1kc{7 z%i#&Zql}qecAxyOBp%mzjPgY(fE!6f5M%?6T7foY#xH9}(O_Oz7MEsO-SMeLx=!)p z);kP9 zhlZ%qA3x?lVJt?+T4rM#(%iGG-+AAEHcjMv&)6-4+uE|;zh4MJy;;VU=g<2=ejiSU zWhN8f+1YstIQ19U&reNB$#&hJNfz=K-1}2$e~yxj-a&*ALnM`H`zYK9%PCdmXL$ z-=(oax7&zZ81}QXF&JAe4m|!V>a`Ph6B|&Mih^nYe+DcS8vj}Y zcVJQ>F`&C`HB}T)*>;EGdv_5~Uw_1Y7p7fzaNZg4>wM3oMB+JUAV2i~as)=;1KUq5 zuiA53PAm&HJgho%^E_}8SPUpYkVtGZTX}N+_4eZ*WASzVPW<6fg~ga#`+q8pPgXj> z95&`$7%dzYR6pvh5EYMQ92~Df5#{M#@mjSFchg-lHnxp5z}VXWWLSoiGxA@`>?{xu z6AeL(VyDC9gQbm5JBne{5jjx#)2o*W18He>0 ziKqYi%Xg2>6@|gw`R*J>4Ww0W z|9ZfDrIuauZn1U}@CpZ6&H@t<5*|WJGy3$>l~71hs-_$PhXOL`QDiRu*O^Bf?(c;_ z$LAjp2%Ya27aaUZjiJnfcmahbb&UY0j~bF4T?6#87(Q!F>A1@(7>>0p>J`I{yMCuR;UMf~DqH&$uEtB#e3m#P_h z|4Yc+UIp%H0c;^J7W>^Ue(pQ2Ou1$sWQ@GhygDW0sKD0U+5@8wL2E+d+yJ)V7t}*>IQBi6 zAx@GcfF``^rqd5FUvo<-L1botZc$#OvDzmY zE7pSR05}^Gz4QeBYhrJR4s<*@K%Q}CGIKI%*eSA7B*z8hdxyy>11eB$`D>s^`bS&7 z5JdSKt7sPbi>l`{|71{pMuHn4G%U#7jQ5!s9AdcUmOJGZqz1pg!-p~dQK;-A2)22* zA)CQWJ4D^9#EPhp{FZzTkndwUVgMGJS%J_mg+G4La!2$MmP~00pz+$?Rm$(f)hs-9 zFP)rXs4B8+Pyn)E!mOZ#I0m2#B_ayI^k{6*%K4i&^jEwk1S%(bxBK%sXE?L~VNc4T z?MPB^7uOw%lKivXTY!!(isXDG#ZC>_Q=~}AN(TTjwFkl6$wb@W*I;)Rry#hDIqxzP z1Aw=C>x3hLF7B7S->?9OQ5a16&n%%sE1fSv^w14Vg82@RFpoqG-kb%+RMj=~Y*Z#? zd8H#*fz!kzxA-2enGQ6c+4XnWm2M>W6`6V1N)eOa8UqD0b3A$+@Eu<})F?Fo-rLGp zi4wTwY=gbt_A{l%h1FW=Oyh50F9OGw7lr~LV4bwQ2u_U?Sf*~yaYxaAMhST#5S3mr zR?o0;Q-03=M$4#{|HSXa$X)4b}ZWLTr7KjD49Zi0m!qnr2EEmoZ*6i z1?JfIF2XAwq6(0#)CVAtl;c>vsYsj8{y>$E#xnd@2mhA?|L?{92GZhJALtGsp}$0# z<0w|C4p^<5--j%C6v>h+TGIXN zLO%Hz00Dm`TL3(#-&|xx!lBCjJNA8-gC>lQI9Aj4FxG^G^-)sqD8VA9H>(2N+}z|v z62R4mhL(Eusvgzh{6vI0*51nLHB*{k7`?e-j#lccqRA?JfW8ou5))vJtAJ`5Xf;J+ zopAV3G9BPIgcf~sAX<2&y|U2?*`9M3U(U7z8NAAO#AtBddqUH78TL zKtN#>X^uo|Pmx+B7EEAi=fs4J%j7Q8jAANa^Es!)nr;p?Q3Pb2z>FyN|F=EOYkM|t zaAkCx`HU>gJ*Q(lQl!7LRH4<4@D(_8N&%Qaw|)ASBhPf}_k=FIBA+ZrsZ9h4y3c2G zMC*EC<+#qCE92c2AGqxN( zeZs#7YCZr|ivM!${P#W`!`iawUDfu~GE znk~avVkvkH@HrGpvo{Qr^ZS)_QHO-i6CDS9!?_wGr+X%wjlF9f9 zKUg9lwDBm|7G8Sbyba({=?Q$j&x~97@w_1&?!LGak!ADn%&Vd7&yeo=0bJ-8Odfp8VmeS(lyGX~BAkke7@j)wj|e`liY(FVHZZS+#&sa& zq>RMMkmEEPfpz=DULtRrP~Ea(^v~!-akJo8^H+vsD5$S)JByS${>->arQP% zz)eYskkdYhhn}k=?7%{vS>|1(fnd02rq7q6Lc<%+-cKGL8n10A4Bf6Tup~l50Eal= zuPt=nUl&I2&Lyg=1BIy^#@A@JEk)u&QuwxX z&Q~sXw?n{V9=uUF!3lK_n%m8>AbgVN>tgsOP@VInO!IJHGM$_l>a)VDsB+%{kYc*PQcOYk`a+q|00% zu0|yzsN`O9$p#{tQT>aSVU9Q1-qF3u{&dZ>t=}D~epEX0KiT?d;dd7FF7U)c^X^>J zwf@KF3bFW&ZaspE2PL!<8>m0HZ&Z8@Z22h2E4cYcxg8VMikNT^S?sqt+$tDn6Oywe z8KMI>eHl|YAvY)ecctGyxmdC?#N4Lmr2V{1cn=Pd`e|-#KY|ge5jgIDAQAgma&12G z`2b76Fbn*HM=gO!KDo<@wi5=pDia%?uLtZ-x8jp`XD5>%94X|HL z?-oNMEzf%LSl_OD+pX-NDfEoByAUslwHGHHb$b$LuaVRTNH?#GbL|ZggnrnfBFRyc z>lk9%h^bl>8N!)0%d&F4?^my-cmA0WQGk(KcfT6lXEK2buAGBx1m+HU?lj!v*sYWO z&N*K%a#`#a+O1c2qUk=`zOe+il8k=v5B}DIkGPJL_Tx_8MgKO9_?8{plS zu5Orm3__4sbErHdk|r|dGy=yI8GCA?Y|>-*;Ax)*s=!pRM*B zt?!J>>g_*2QJ?RR5;~eWdp_E2g4s)>O=__)s{~=sk*&EviAL**fPpI~!|_BWjcs)H z;IJa%-Jhi_UksE!XU$hQ!XPR~zmMPrgD~<2_S52v-TA$nmwRsKQPQN%9`u^bmRLm0 zxvoUnKAP~}bs|4vu+=wwMeuim6!n_@oGeKVM~MfsFi@!b)7krqR&wP@V`JzOspdGY z$-Ye%lei!2ZL;VD@=fHBH2Ctl?|Ito%wifR#0GBNDP4WL=Pk5;(y28o1+ zpOS>!+u9A`OB;f;6S&yt7{aa9#}@dion0%F@ecvD!n15ApJi9y_NT*)Qd~Y0AA9wq z5@pP3I{0ww=@NI@3rPq58AzU~WKky=j-FjGp1mVtmWCGT9Dh*zP+)0xPRbj z0=%Q~OA%epe8T;%au4NW?;PWgv2EQo{>%q>DM}=wuNDV(vM7#HE^Y+(njsG~?bkBw zMka>am&Oh3j%%qtBw9d{bF^~a&tluPt9Hs2ES}g6U&(By@Oxa<(pRhytvAoxh4YbB zMr5(W*~)EIjbx4Ub)XaSCsdk?#LT;-=tvJaqEv ztBZWqjk<1$rrgAbJq|rBdNXqbr-CbayCwLit&YFGo}{YxJVwn! zPhnngr0y;p%Z*3An*!0&mFuKOh!ph0lsXr26t*b0>Q{a+S@vkmtW|M^>Fb&DTY8jn zI&uI04k{QyH5cEW@>DI5Wwc){Yl9GR5@SWio{wt3?$y+A_|P!gte~dLNe0z?vSbf# zUyeEhw^+)k%U83MNJ@tv+hhTOh%GM`c^3kVzkI+r>Nu1MTuHIav}I@vlIA?X3uU1m%u<@U&j4Vd~V^z;4^F44CZK@mWC6}q18F_ECw$BuZ_DE ze@r-)aTb^|+hIwa7S)KbC+*Lz*w<{$YT*|%(Mv#JqfGVfBX!>(mYhx@)6BT1wMAjA zblGE4W48i}u2oD~a}3^hVJrP*(R{nM>O{fnl0^xHd(pm~HFUPtDbrd^e9}_E2&FtR z(f!w|45&s1a`n4cxti5Y4Qiq0_T~*NV09F>x?3KenDVl%#-x~E>rcNnn)$Z+W2dHw z`L&*#xh0cU31yW!DtM#zMe0f1wnxpe!e}gXm`a`NKnqHpYggTfE%CU^-@V2-?Nm~t z7x%aCHz6BxR^78TjL>*uGQ0VS{N6zczf6|rtSyTs*R>YD`$!PvCspGGf8E+taNCf4 z%#f;QeIz}uW8w<_xAB?d-0PzU^PP5xtFzek>thlFX?x9=v8cG^dfF6iZqa8;nIr`K zsHUE`(%ro~w5nm1g${a>SvjdfD z(XKBSPG#{Q!&T}m>Nu;J(QVL@y7eO#CGJ1$fvOZ{z2lkDA8kC`Qisqy>;!ss-A2zb}Bp}#T<1m z>>tK}Q&3-8;S+5!CxYP|T=hPIenfZ{XK~7OzUo|eGSAIan>;%4p{JchJ9n>*3Wa+0 z_u27lKacZ$Zl=<4-=l)qpIh#w_YW0xiO&0ZA?su*r_&GWjAuw8E**LU5Ee~`RTAU% zE^wm5G;yk-ZjydemIPx@;$CoE9m#p;D68>}oG#r)KwhoW_Eb0>x{vb}`H-Qu#g23^ z6V|RmrC4{7sQrMCPO)nz&j;5+q*dQKtnc4^Gs>y&+kAP11r1Y~ab67*E(=R0b~h3v zTXBarHtUT8-eyZrop#6=CchE#e8eRa!pDp z5s=U|%I_G9#k&U8GdU%B=BV~n>xf?A7{x^Sgs{6)D7bM2i`CUUIIFZ!fkX}03!%F@?S*oi9U<%OmRm@ zayjby5{7s8?j7`>F#@lnKSh3(Y{-_p=d8+_D#goH<%w+iWKOjod+mX}#3!9-=hZ8q zshc&0T;bk8A{+@8B1g#iQH+@-3yF2~79RiUI`7e@0$Olo{Nb^qcge*X+?1o!C1)R6 z3+-A!Jo8I~-yM@=)a(mal#!LS8BXu=I_FJ>sotSxrnH<_~k;xc)Iz9HO zGS9DI@tF_M^Qk(XRuiqPK*-j#J2S?xcy{TS)gb>#1> z(Mv~~!+IG^tcw*Ey*JLGjZq8zXNp&j4vubZ?i>1Jt4|VS^&DwceaP)?YMWeN$7p=U?&yCm#3pEFO@W+XyTh^4FfE)!L% zYyH^1#8Ql#H=SJa%M1GYAMI6BDkL}%fx|Qd>D5`o`#D`|YPdCCI`W_MZ*myNT7*5S z9Xli#;1#su;}C%7g2PDJYbH&*S)!3f8AW4)ec==k<%sYkdboQNnkaY1n^iXwhy@nhuMPfg4FZfg z3D2QcppSnCVO5e__Ynianatxf9UPy6S01P+mh)czRekCG7V$x1en~NF%NgQXK-byQ z=O34+x=;u>($dIKv$7Xw^_K>X@a8K#=-l~NFeAe`95J_uPf2XN9ywkn{j@Jo`W{2l zoDppD%>2*~hJ&46SfYJ)8*R?%)j)M~&Ea&ei7iWtoJ^DURZ{GT*-Dwn;+4%q3AIYh zEjkiWy7yC$G&fVQbzwih?!g%j$+D2xy1aM+^`WY}xWw-0-5 zR=4C9Ef2b+kMrz@-ST5_Z<;L9BQn((sk*lmcPXBtq!o=V@l_k3g4ouP`8YZu@t*pe z9|vR{c@k~XvRZ{>YJ$96->_flkl^Z@43Ij}Djk2sNsqJAW{2-Ho*#IB01GnGuoU>{ z*|rEYIDyB*2m*{oQy8sIf?qsg{dMLv^W<1cc}Dma2e+8q)#VE)k1>(>s^vxuT6$}1 zf>dwgUswg=Y7u`F`=QL3B`YD&sP^Z7j^o2pgy#Yn_>W0WYlq+hAtZl+^ie3vYYFbCvzO7KVQZJ0Qgjflnze0@*qmnyW4@~m zYRoq{C|%WUy0BIGzNy99gw?T|F z`V+Is7m$O&ct7=aY>s~w!~2te_8ZRL;Hwt51rpNO{VVXpmtg{c&t=@=bI)#7yCi#< zz9GcM=8%xv2GcbRCGx2q6e32O)C^dZJR(V-3hTTqo3-i6Dx7m(!Ha)8eypgdC~P)X zxKC45L}DR>C$~`|oVLAwqk0dEdfTwzE>$uNoGlOL#}U{&GjEFTt-nMBT9AWVZXc`H zzIma{a~99HOUv}CZZyM^NdqFdf2b-8J2pYhSNsxaFmZ03wG+;LhO?RGuR= z*6TE*Hm^0v2I%aTCOJ zs;cOnKM{|=bnOrM&F@*B>c(64F}>(Y3ifnLnmf#>r_m@xwe_R1q(2`X0(w_&V|U{g%e1%#QcD( zg5GO?!R^%u@2!ab^x3mC?%f(+O37FOaR2Q{wma37D%>V?H&)Axtg=#h)`36ht`#%w^5N=jaH%O9Gv>-uTH29$a2AXYaxUmnlW)C2 z#sQOOB}4-{@2Y9t?n7V$&Pm`$Ps-}Bvm2x@P$-=;G1MA=6`Ef1-l2*@b zYu*=(ouolDUOHhyKCsX?29m+F5C*{AgQ_GEsD$VSHuKFXY3wizp3fu!^Pln&%pC+n zH+ZQ0zDR>7_1{gt0S~@1!XGDtvn7nIfkjw-PX6@P$2|wOZ~amhtZLa#P>E9BLq?CY z$B`_SWH;D^ITye9qf1|8?XI4PDAYFD1z=m5@WvykDpuOOc53 z0KGa_JSM?yp-BYHca(cG!U~mqNy$gSXC!C=#gD{IWltmPG@*uc0yuAmiW)(Gxw&ab@6)u(7p)frpNJlN?NLljx+Y z8Xj|J&YT@-=Z3=1CgDFnZXt4obvJu0H^YMkVNH!YTUjYGORn05xBbX?j7yt-MsCf4ufZ(7DJKn$-GiDjSu6j@| zbToj;t*``piNS<%(sdeuxud4%FC@gDE_BBFf#k@Q27Y1kMm`OMP7|FxDTsuW|xegP^zLep5rO2{u950X`@}# ztT)T(y(V5ESaZO_8u8L@%XtEfOWe20Z~*ghAlA}a?$itFj)@3%m2*_vNz|myLHHxgJ;reGXUzxgx1_3@n?Hc# zm*?I@hwF#)*AZcJv=XffYx{S}fX~8-UHJZkXBlOgM?^-rVxPhtnwbeAyBgIaE5 z&oJYA(@mCaYtw)FMV-MR?4!b zxnz2zcCTofH-l{MAk2SK+Gq0>Q?4r#hM`TA>7!DT3M)sKpF^c1HKcRnJvF{TcYI+G z&R)Ry)Ve$(W9cWIVxKj~!vFce4jSLx!))6!@SV6t;)Z{DcuK$On8*kGOy* z7KeTxj)uzsJOVF7F%Al4>QG5|dRALWeRMjiZ(VK!@Or=eFto;8jD|ZnV_$!iR2)N; zo)Gpl`-BdEj1FO9`Xi{(NI#o1UIP)}1Mq?WZs}<5RUbXHKKcV*)fIYn14?pPIMcRLkJ3H_ zrP-}n=I63%gyJt7$XPl_qLv6P zPO{1G9iE{@CtwHQtPQpYIQ|vR%*HHI4{y$q`n1YKJDF;X#9WqFsW(I5QnUI$g zc@xKWC-&)@v{%iqt_bUNZg56QgfLYV*ElS8E;5VU)xTh{WQm-}<{l$rnX(l3 z+gjg+Y1tO-%y|Ul+!ac}z44v{Hv!v~jjZl^V(wp~zjvLa&N;;5=nL+a@nIM*<$WMF zI+}NFy%R#*?Ahf3>={08Gn&zo6$UbqRm)5!qSYt3vNeYk{sA&j438>%N$;a`LRnX9 zhpTzg@uXZAjWpKkoesP6Z9aGBmCjX%)KNA3TmgDjq4*?(guP0Cny{7HsO(#=RTztGfT04?fHf7;ua9y$^w zLSPHe$**ytZp2UL#PAj{KmPn?*s$&>t=f3gYLLpt#okmZ9I<64&z<1Q9q6Lv64YrO zv%zQ3T8d&Z>1P6LvH`yP$yjKCDUbZn;VYZ`4=Kk}5Y9YpBQm zaHb6e+6OgV3{&XhhJj;gx(imZebSfYZgCfjsr!(#Qz?kPHW)oO6^-yXp}ZvTdx9~0 z{i>^Rr6$R#udanv`ISR zd{Jj^N6`JP|30&*fci1D3G!~4_*A0s!Y+p@U*T^sDN6uGYUayjQZb`L?;PxynX?-FC0MmqP_%#nzuS!yg4n{u7_x#pqar z&{V_z*qJ(!*g|r(`u{%8d<4b!j&a&o9`~fw9vk&m?K0x78V+t;uxDPmR%XB5?zheF z_z-kuYDN{wetO?*T|`WwW_&gpjV26)N5Y3#Blm$#Or?e;0=@jLg!K4>IIJ0a%wV5x z$M$=1DH$@7Z@aHQ%6M|;VzW^frgV?sO;=a#l7Z%cS!nYL2G;JC=RZC3SuBIQds{KRYEK_wn>1^g?+6WJV9l*r|9RM}Rh+JGXw+m8Av4~6)2H9r_Gp

J66hXO_IZj@jNvB=uN!#+;MoFEf65^lIj$eS*+S2yX7 zMB-|)B4wlP9q5`x2;S=4X4T*~O=zaD7!$$Cplp3Mb2I-(Cy~8$XHJ{ukUhdOx*x^6 zp{0CDv};wH1x#}?WlVDcF-f<*kI_l~lG56CgZ`y@VdHJVtC@V#`>@_rjE*gfU&2f2 zQQu0u;^aN+aAdXEm3Kc4j%Q0XlO}%E$W>Jw{2aD%puI?4-uLTQfp84pyLWuQC`5Ur zJu4{dTZs#?SyE`sX70Ya{FUmT-=_K`^?!;@6AL>u8zDlc`!t$gJu2w0w$gRmNpEZLLm} z@ISQbDv83+4Q_jyMw@)Oy(8|6)}V#lxf5f><>!wke$~z}fJgurNdePRNO7qrSSC5@ zV7Z2YCy}5t#}!%;x2D!=#v1tbZc61J@1}2g2dx`ix&aoi+` zCCJ1=>jlT4rWWT1s*GYk{*g}OE_yWV&p^Y zBRX;|(q7A#X|R%n8z-)aAU098>Z14V;#yq32%7dFHk@RDD^C#~RJH(4$ANOsVHeu4 zP5xEM^ZPAaP~%{+Cs8!qYVnvWhah~t1(QA@h4DV*5JN++ar0r%R zpT%4)qS`Vqf2TAC*k$rOu;%}2;-TEViTAftF~1NQZb6t{cXsdaJki^Tr{I2lQO$7* zBM!rFb~e&~SR)n)hSGN(zJu&Th!OsZTP*=v%+am=d5l6aW!cyC(48%sF*xxAw1eM zwg^M>;Y2wijV(Z%+*!=bXQ~8a*Ss2}uq3_k(~<6lkLV=@PSLG!rk@B@PA4%>Yj?1h zGGwRsW3_L4=?>YuzTJ|uRK(C0{*1rQDyt4WCDI>mhY)xlB8 zBFA}J#VW?URfwVD+|<-2?a`Ww^n9V6m7{!MA!U&8^BFXw_j-H~CaDBnj>&U`e^(a= zH!Y<%;amwa8*Q%G!rMS<2j!tiHUg7{_s7#b7c}>yCX%-g9Ep!4m2^;>P+Bwf*xh%ogZ=C{2*E#d3&0nj9|$B)aB?- z%l4fv(rpL%+O3-YkwV5!bBcS1Cz_vF-JKB|IrZ+o-VW;DE;wm%w@e^9*MaK89VUq0 z6wJT;kn}CLBb1;c14g4968^%XQ_u1wB@AC0H0D8HRnDm}vLWg_D&N3mI=&RkVSWRuqCPkd(< z3gT#?IJ3^oMmXVYLw~+5VuG_BeZH?|N%bi}W|xv4!Ww_pBdSKfOU=R1n{Dl@ zCLeW4=wiKzA77@EF!yN+{ILKY$0t+ppa@icDEIGk2-;?9g?zVIBN6Hu zvrOksx%xm&hG9CGc}LT3QSj)fe*Sw7e_tt^{cWRz=xxe$RsSa!(_RMt$xCqG7po=0 zs<6H+u`sN?AR{h7(AqhtnFa|CL-oz(sdNUexwgSBcH4Y(N23*#lB%!D$3Nz5|8z>y zdmO9a-_$`Q#5n#a%@N*Sgps4GCtX6cIxD#dN%E=u1pQ=j9p{teB+(J!N_#j1oo>xeG`z<;@#htnntB<{0*#~UV!E~BDW%=z7o3q) zt!P!5D$k3x( zzEQ9G5c$aqg5jM`yX=#`%?+vZR|bO+o4Os}EW;RSALozgy*bD&mrIcrN6Q?8Vh@~2 z%9u`TEgZIeq0Z*y4tZU!J=9YMeEH)k=Rb$@rIO4(*}Y~rQ`FJ=@h#RZXLsir%SOt8 zyKsU;cdz%Ooe-a!>m@h*v6AwI_>=>tttczE$Zm)2^yU=S_-nVtm+_Z}Zgp21K@|=P zL8=4aWNm+P0Z*nHa51yGGx|T7W+h#0CsL`aDZXD%(n^XF@E$L(zyAIsOLrV%~0<>8mQ#-TfPwQY-|t=q<5ek^wqq1 z+cHyI!^!JB8#mJMiq!p?aCw7mqq)=~@{8ow^?|cx0C=7QeO0jW&FYFK#e3APycD4w zbOKQ*LeLohWPHlbih_BAmH;gJDe)I&8-zN3p2PA30p?$uByFC}Iufx|Z)8EObJk zWr%)X5ZIr@hkxLYMe`>f8zIeK+jodTz1=lOWPR^1_XV zKE7TgOrKd`!_3;OdloH#-J4Lg$4=$+=>|&^vgtNe#R544he5xtAn~-L+MFtDn!~iB zLax!F$J%3*LEmHSF%xq}d6weUi;3yIVh~`wxoB~85WZz>Vz+;0YH4NU_8D=o7B z`H06_b@nsN%5X_Ma{B%ve$;oO(iyVPIpmJHv{3;FDiNrIJM2_28uo0zq|@?jk>AsQ zgaBBJ1lCI|a+27+%|%nvN$r~RmlI@D!5klli!fvGXJb>O+Ffa4NXEjrSPM$RuV|PQ z7CeXuSx*txER}D<*q#g%S)$(yh~y-L4Pp?Y^>M8g~}2i-ZnJcaStKV&P9! zM``W(T6tz&xj3e#XNTHpWA~PQwm9R-!xgKwKAy^8C zX#IIk1q|(9sj?442^@bf0k|F1!;qD%k&+tO=4Vfw-5Yg~uR0e#ai2dw+0i}z=)$QA zA;IUz*drwwtd0wcMK)ZD*zTLAIdZnod>)O)-fcFziej_DvJd%THhg;R7)z1ZH7*oH zopA{Bir?;sB9cKTe<``%1RTje8k{9*=2?QQ^1Z_&0}9A8N|L(p%_h~j_;qI+MTM!l zUF5yH7o@!L-$~kO$K_RexWYYZedq}Xl~OJx#;hOqZf0t?huUlqTF2DAO#Dfl(jd!~8X)C!5}#(Vm_((njvhZ;r3G zvxhtQrC3(!#@#b!t;sHi`_`_!$*oW zaY~#_YxiZ=xAjg=SE=JliK1rk-9TLo7g%#vC1VPD9_QayWvVpOt%ZKWH-vbdYtb2= z2**(Mpwq~eD#^mA%dRyVV_zqvh@0YM<{Ap3!5nb-t8QM+UH84rl_NCL)X=_Hj0I61 z_PEaUgmRKAi}*+Uy3C)kb?-$hn!C-I-Y`dxiIp{TdKt^)afW=qGcWrz2ecG_H5?6FqB0Yd%8IYC%vZ-tgjEy z%r-74cB9DRlaz;lEGZ>)tEk;OtR@R`uwo0qnDS%>t*m?BuDh=0n zSk0UeM9Zq!ZtZKo@3D0CMg?yB%8l&0bSoPB(y!jlCDcaDVW&ePqUx0;7;ys~W&3eG ziMHt=u2Qs!KhJ0Asxb3;K3@vATkxaRLCsSpeZ8Oxn*Bum%{-}Ec&@)WZg@IsJtZy) zyUz0LA7Znc>XX@U^fsuUOPc<^q?{y9G|h^;HK;r!L2@Hd&CeD=$>$W5p=0@MOd}d| zp%5o35KsNaa%cT~-*bqh=lYPBVKD!AOeBhlJ%j##H`3&z3(ClF18tssBJB5B>C2I> z3OHS-A>%{z`%f#ey5Jp5gmZ$vH7)b7W4p3iJ33+aX=&2WDeS&^Qv6=Wk{r^!`V$pW z;KgwHS#){Uvg&mBCs$`CSalrnxi5Noh^Jq)^5@8EA-A^YfLej-oyc#N==Tn7bsrXE zw#nbga235Vt+ePQzAAB4XxH8=Yj-6JIE`j8s~%xkuiBZQ)7^U6(EwLoW0_gfE`O)P z+<&%hiM5rBBmL7N6Mk*g96l3GwTmv`ZC~-MmT7}TC@8&T>Itw|$+^E^SrXjgad`79 zqu_51^gJH#_+U7h#(5b$BoCy^HDp(onJxy4)@}qPe}2l4nKEeOd-*uFgnJ7i685~0 zA!qpHKoioYLq$?Wg0emrrgip%+hseHq5VK%%#68X;%K=W4Qj(3?v^x~6PtNV&6oOz zRMYf>HUBVSIYv+NWSoyIDrn~O0{o7JnBxuy_*Tg^Q5ug|Ya>byiDoiN`O2$8cqszB%ZnlW8- ztC#ly%Eh}E$7*r^WW_1y;mW5L3Id+QlS-0Z=wo|NdhB=S5-Nl61Dg@FJdC;b!^E3B zo@jjbfaj&A+@7K|q_@NDJx!bt@lAo=lLyC@q{(9kcx;|toxk(V%~RQ`)j(Jx9264{{@!Ngk%ed_OU5h3#)2N!Yr1cqpPhW<67RRyvbkUZ)R%Z^wg9nlk!WDf?Jr+iD?Q-4 z0Y)vu8)>{oZF~Ach(A;Iswx|;^_H1JmeH7swCW&ru-Sbgo_i;09_r|3a zCML^rkb2+2)he3jw0^(zI%RCady1Tp zpYpI|hW~p3Z(xAT6a{K#o$jqbTm(UtpGZKewn;|Z9BrS)*yQnbxBpY&^ zF5%g5a?V<}TUveehWuggx$Z-zN$vXdY`eD4d=<5sA#*nlI2I2j)^VM3NU4s7gs1mIp1r=VDV~#euXDBU(W5w0f z@&sTDr3ZPKMuDfZT*W7Pj@T-Iv5o6p_XLq`ydUmZ_avm9e=W#?(88lQqsnLBi|(#t zbG3}T>lV=zTiiD!3&2uM@wO6Kj-`Sz^shrlar5lBg!+@(PmG3q~`4T zvGd}5xPO7$t5oDrd{Z7 zV_cHG%C=ICTs_p>o-t)pkgsFOknBSEv{)zxTxh&(pE_MDxHl0%V#%pbTjX<|V=Zhj z$Y0Z~iHzcr^uljU--K^_R|^m)54G%``_je|jP~G^8&v01Eox%ce#1kxedqNF>{g+i z*St1~t0YcL&}lN(C%~UcOS>-a)-P>=#D#5xm_odUG3 zmZ{uP;V1dw$}`bt6;DKKFC|4nP6z%JKH*duIT*S~IXy$&F4Zi#+g!eNpgh{pJ{^0Z zMz>4fQUUS{Ij&@Oz%CKsQgXv!6$jh-#bn&zGb}8`eV5RSr_cbD+xO{qn;*#(; z4^A6Pq4LZwAlN=~)Iws?vd(CT`B0x~38aBdv4w)tSPuKSRU_$w^o_iFjn~R8Nc*sd zO4ijXmY4c%J7gEESs1hU!Q9j8ZJE0D?H7ziA_hJ>V|fek2)~F#PbAKqMT$U$S7{oL ziKB&fdd8)N$!=;vNHW5_A}ZYc<~1!moBOVfEdqKk<$G!pvI3D2=Ck~jA;)0O(HH~t z-k`2jo-?H%Png<18=@Tetn; z1Ph0S2tR>G(5=~+pphKs{XG5dC4K^R?9E=(irq&ico+Wtep`k8=in5Jm!6qw{b7yf z!>EvPFY%ZBtZ&?G6? z35ku-As(w{;dQ>kQ;fCJB=2tGyRb%`g`}Js_hmo)CxL_otI@kx$11q1-v3?ne7x>+ zZp?JrO9GF(Aw!AZ9B$)pR(mvJHas3AQMX>1m7s&_$BIANnCH|`|BMX_+`&44vNaxQ z6K(E14yi(m&OZA7XvsnRi5~D1Ed7*I?5%|2#$Rl)OdCr6Nh+$F%K_4xqM?VjlKtH6 zh@0no<3rWDoG?v!KIf(1Xf{T!-segLyAATG_gWIA(E&zznJ6ew4hHL98P@JcVN^Rm z@KF`>BYFHo9MrDy!deP;Vx?>jmnG2AiS;)eKijv@|XWf&mfR0Di3J>ula&*ST z*U(?TH&7As5qjCM88xpOdoC>`6bJH=23&W*WMS+HYV3Gld2n!;PF23qBfgHvgy`FV zOv`az-7FziJ&KzeL&ffFqe4ZT<-A(YwmS9AMPz^J4QXLQprg`q)mu%x+5JK0E9>oy z3$~oWCj+dd_vg2U$G5CGkxv~+8&*1iTU}*BFeJkE@baPo)hZ*Nzj@B>B{X~ijwDxB zHjEy=Z#xBe$S;dq8?4pic6&%pHJGD2LZ_6QAAQ^zvz!v;xVhkk=uT;rhp|%fy=6My z9T8=?aNjrOTIrz5qh%+JElLm23+cnrPP(dROBo2T6QCUh z#)}3sS9eFyMSI=`C7qSs^5?H?fZ$KDs7i(2#THQ+evPK`<6P|D%h6>ww`Q4`QRCiKv9KkN}EMl#I_*6H;v3)%|HKZzrGJdbzt>SG*!^+Vm{QU3<(HNvlL<;{q z-J<3y#2MVJGQ3Kg0kC42gl9&qT)o-!deXW`0sHqx2I+@UG@@$`fd$+0IzYlm|Eq+3 zxv_@2ETww+QLjJOw4EHKUC!KJ($JmJ4?~ImtW67K z!*7}S&AzvLTy_*X4%(%X6xwJ!;2o_b_Xeh3T3r%0o~Ktyv@_ZCkm{lL=B4h+q=3%) zTf3cmK9kttdfzuPhstyLl57PCx-s`sXSl5MI9aN;bt<0kC)HR*FA}yX6?>6L^GF~ZDY=G| zEr76z-(xM~PQj)V?XG$T^rFApU!|G*T+t+*osI{CExdbm^ z!Ttg_m61cFlJ?&nm=cxrYv;cV;lEV=Mm}-(HmV0xNTXSZB~~^n{_|JOW zTybduZ1%u0_Ug%wCqK|XOXwB`lwW1!J^X3~q11Bjnb7+-IMl48@3=I125(S5ZO;gC z>y^Llk#^>`JOmUuDi1}NJ3cxN*AH_6$_fS=*#fnHty1BuLmPezu>IKzn#%mk_Gb!w zjz4Y=sQ;ppF0%mDLZ+ZuO1;I)FMeZ#9+%th=-U6% zPZrR`=xuD5oQ@$G3M*@4`*C@^@{z8Xg6g_1!tniMRcKbOygleplQ9F#Kz+crhwD?j z`heFJ>{JrQQf}GEe1coV-{8oyIm?ye5bQ{U`~LWJ4yT;J62lXll^vLZke?2p0IgV7 z^BOJnVXV12^{%2!Y+3g@<%SHPeb^42eP)+mErMcpcM3*2Lw%c9$KzJH`4!K@>5ca5e<)>$Pc-TPZn0{S3Up) zz~EDf8vuw6u25pb30Ix~Be*eD%}c-@TFx8J->iKFJzS!P7J4EA>|cA>Z%<$twXXTS zs~nm+O8?0m(RnV{_>M({x-WhA(4 zrX(|K4##3KHGa@u!vA+3g+?`;t^R(ia8C+*b99=tXO3*Z;Xm?kaX_Bh)U*G8$=lHA zggcDv42=^lL-Kvwh(B_|Jg@XBMOVLNA=4!Tim73D{;g5aX0w z8B-+*2md5t1JSQ@Lo5hKLxViRchUzD$u3@3Q*8PQq#_(~IXBQBxDkDv-Xj9B57i5B zs^I_Z-28J$1t2si078SuEe7#Iz8=TaURwm&d>{y=i(c~ z(P(~Qz9xURDgTVe0D#0cP^~J5>;?R$O$AQ8*&(x=BE2~K)K^JjvC;fW9a8f;r_~Q@ z*RFJ>wG1$>suo^`M$xw<#t&vFLn*m`y}TR!ysEaJ4Mp?l?nqx8KAkz2?+Gvfv_bRM z2z+=SPBx7}8tBfDzg_~z2*y1r6c+owN*NAD3N<~Sz*D`~jP8M7!9#`R`B>)ka0QEVbFa6W&^XbFyN=+}OzSb8J(en9yowWozYuQ0L zkDc%>U>E+=OJ|QFwa6o>;=Ed&d*aW5F#qkn&cPMV)oR=mn{Y^{4Sff1Cc!$HetSzA z{%2{#cvby6rDI2D5S(9NpYun6Ubn2zzmM3J;f}>Dd}d$m+*C`nm|V>%KOISQC@Afq zz!UyW!ne<#a78rUqKaU@Fg1f$4@rBQazXoF-%G$pwsT5TNMz0z|J9wO?7s@a;@4Jt zhtHFOSi7gW|9!y2c2D4pwB;>H=8ZI5$QZ1{~?K9v6*{0cJHu zpbY~TxcYZd{QAEu72xNx5{`E*yPKY*qe{e{1 zb^$!iO?~jMKlXv{-bU#JcKQF8Uw%9bSJ7=VWOx<&a+>VAgdAy2&gn<_2C-UEk^k&X4Q!fN*lt?_2&n<(i+n9Dc+9`ITQw}xipIsuMEFECiKU=dVpGbpPlOtUQc4Th0w4$^SJZir7T*WLIi+1Evmr@V& znUJPX>LHHq^prGIT*YVMg_%K3m097_Thk}4bJcO(%ui=lY+PM&Wd^E$631~WNx0!> zn>@P5Y&fBmvp~E=*V#IB*w^@Kgc%a#J_V7&3XTROmbHj>q^)z0`Voqh2#G4W&L>ys zei!_tzOSc_+*k%eN7*0_>&n6Cpcr(F*PZrLaQM#}7c))NzgLcv7l_(*Nuktqc7WUS zlo=&M02B8CN`9q~T03PyQ}Xa84LiC4&rs#&yUog=j)F4<8g0``j=I4Mmc$98nQJl3 zcS!ZhObQk8O1g=UgqsahL~&_z@bnDQ)pGZBGHgEaRrbxdZ|+|)LRjm5FsV{r8LQ@= zKg|FH{L14f-Qz^a$265%OZ=*1JX3q~LU*9JQLE4I(^KGPdK+A*$x?nyp0JM1O@r+f zG}Nj)$2e7qqO>O_9qH}?T-TG_uL}LI(fT9BP-NWj-SIVlYKVr!1mz^uX;~K z`rlq$K9OE?>eVz^xexO1T{cQp?UPL3ToYzrCAv3$vCq$*oevML57WQzuP$}-@n~SR8F(3&93~=gUsj6@4M%=zXEDdZz9Zk z*vj|I_0M0klpQ}(fK6M_Hn>;%X`%ixS-W5LtzT8yQ#bLv!oxqD^UCyo?7LC9LT;aa znG#`^L|n#`60X)_9`X4-z*V5lf+oWIFT8VS`Ok3UC(uLB7ghny=tQuV{QnJH>0<#B7xNXmUdT}(+e$W1z_HD1CQ(gsLJhg3( z*Znr_a*zM8WC&Ytxj`PK>+|L-|tbN|)t zYwNErzt`Nm^5XJ})CBpbvQf%Y9vFX`U6XEhZTrmgCsd_XyIfBn-ur*%g7oK)PQ*N( z8+p-APyY051+jN^i;ItQdMR_i?{)oACs8@Y|M1oCmakf@|JypAKX>%jrTW{T3a>Lv9 zi{|H-hhMtg8r3(^4_Jacp1yI{Ci~@fz3b*AFI@IpS`=tZ?p(3e;;;1+{gm}qo&KAD z!M|q1Cau)f(o+>?FxSjc4rX(8Pp_-&)i?ZY;`4ZVV{Mze#QO_jx$4UI<#Ix=6idz3 zp8u>OrBqu?E^iNThu7mY*E)-1S@|W0e;NNa`#f{%m$HX{e^_tHo4IHsa6}2{=l+|^ zFP?j0{q5z3|5xYc*?;Q(cuwbi@H68g!`m|bJ+6-**7cv@3caVaKVEe8Yz_I-*{9WK zcXCgOD-W}rzD8nZ`Rse=uAOL{z0v;SU87?PpGtS1yLRPtr_QAJXQJMI{nfU5y0qV$ z%Xf}voPX%|N%q#N{98sqTW`)Uo}ad}XZFuUrOt29t*SB#4319SV=R3-^>pSG$=G8x z(~PhDVo{C&mZft)o?COVyk`EzzMdT)D)x!BhiUc)dfryKKQp!VCF8@*c8g1Ut&GyP zh>0iown_N`%bDCyb0ZfsKQ_J=az1m4=KPHxTcp0M1I0twwhJ%j)E@g~7IfoS@)2OE z{OfJcI$)T;jXM6xE?IzWr{W8J9LF$PS_%Rjz;paLslA%_;le9J1|aZs^>bP0l+XkK DL&bck literal 0 HcmV?d00001 diff --git a/mt_with_external_memory/image/neural_turing_machine_arch.png b/mt_with_external_memory/image/neural_turing_machine_arch.png new file mode 100644 index 0000000000000000000000000000000000000000..92076ee60d29794716e5749125a2a31ae13da359 GIT binary patch literal 34465 zcmb4qby!sE`ZkJ!v`9*-2!l#W*B~GuQYzBTNY~I^q96i8Nk~hGG(&g8NOun14Z}zy zd~0;?efBwLpWk(T{^7;Ita#t`#C_k-y@FmS$r9ny;A3H75y?G!`U(r{3Jm!9;sy@z zKl{rCJS;5y9CInD7jjZkcV5`rnwVP|V_`iDijKv7t)fYT*!AG7>%V&A8D5-i18{;kfr8zI*zNMDdsXRr!VK52ieQN(rQn}G!Gn-?#}n%;i>Rww?B`>G>;rJgV%U;e2epLTHjn|;-M2jn3@Q>!)&`(uHjaviDF^-H5=hAV%-T#vvU^~AzBVo%Ta71IBcD*RUR6!S`{7GEnMC1UpX4jDhjVd<`l&CU zFHz+kyc-fY8YJbuI^iVF{y$T%ee!?)0$a&fCG~b9Zfuhil-AMzY>u1y8q*vFH7;Wl z`85gG5BLLjq@NKD1`ytL_;5w&$xb*Y`3>nOFJI8L20ra~H>3STLLtfA|MDB<>2+aT z-2fX&v3@DD8#Mki68Z+$zBU#Y;pbr|G!l;AKJ@dSAA!;(UNvnI+Ec2eF7OR&QQR{= z=J#Y|c&Fb)cH_zS529%>>br!k6e!ty5gJcFSUuejzn5{<{E1cs@e;@554AF22$Y&8 z0LI-sZdl88mztmDdI_nD?}MgcgLp1=ZWnH;IJ<5aHj3O3zSc>j&*M-lI)3bgVD@G; zNS{%m;VwkBh|x|kQB;Y_G!)$|J-1-D3B6t^RVkxGTYxJNlF+z_`sgZED>lQ}*PJ!) zfWk!~P_cV;$E4?=tDjS4xInHq@8Nkit4Wf6fk7ks?mZ_h3o3u=b1yJXdP+L) zUiq!vo3bBN?wdz)bR?52d<0qFrHG<=HuelVv_G|haGQP-onQ6KFOuaAQOz+54Icj+>X2Dwt_n$Mz z-*;!8d3KArj5&#U?3sx?rCf4MEz^nCN7hWH<%eyNldP%_os@F)EhOVvtXZ}>lkV%@ z(FJ*_xWC5XOlMl;Xo={E$o^Dyk4HhuES^`LCZRl3O}a*gHlnR#qw}%xqA}W7`MYxm ztgGWYdo)3G*QZC(HFpG9!@h5R=UFCOcIilo+>N>{er6njPK%Qkq2y!R(B;i8(rA*q zD|1)*%gg!tW-96Gv>Kzg3k+>0ZJ_7W%-ER}&u>2uj4pUkr^i+^Zo}KmRs;zhz{I0f6NQvJ2Ee~gjr^rpP1*5<&3#! zC=Q#LIh2u@xtJ+$tZlqow;D4ay*;AZZ@gi$Il4)__VR%4;MI?ZKX%p^f80LBI}Q9% zLu7z0k6j><-9#@n-rpH+>iWIN6wf)R=hj8!N9Lb0dNj_K~{cOh;N{l)zDo_1S> zQiXephjET^o&sgOioE%cy{uRuE7s*Bc_Uj%RenOq+Tru$>m$uAMp4qiZOF_3c}NU{ z-f3f`$+~IPbn*0lLUdVle@>G{w^o;MXLWaV%H6IT2IO2e;|A6`R{Rr*6(?4f#t9}K zorlUIOf}bPglh(jtGkS1RM~0E$BIlR7|R$PQ$?N%(b-WuzIJ&dq$UV<7<60^R28X% zj=I2Hz&q#dZNt)Q_M^swBco3KmI<^AE9)_i*{G__2^NrKT|_pQryapD02()Xo-&jSkZ z^NErO9S9x~(h>^b&EE7Dk$mu$YT#B-7-480EKy^<-u|*PLNzAWUi~0{R5z*WbcH69 zFUXQ3FIuNd%R?BSmV#21>L7Z#FULUfDNlM=8e*bg3wukQ4%7?o)d=xYFzgQ2I2&&s z9}qmNth0mY$l8z|@%2TkH+#_TQs^^?f1XQ$vKb0bNY&EioFUu zq}2HK*Zm)qKR8oqe-Ccnc+KkMWFa5VW&stJc28q`Dz#3$p1Gd#SdvG{^ttIik4On& zvGbd{?}y(n+~`f{J#^394-T|PNvxqWP)DXxZg*QiOQtI2qAV6Kxx>|LL;K zR~NboDoOe16F+miro=LKq@bn7uI8lX>`-tg`@tuUZVtMc&58%MD^25?)uE{S~q8WL~z`IsqArw7(-Djdj{{B9o{z8w_}%?H>=@1O5;%DT!j z&>e_QzW$=}@^w(ILZPP~h1Lj)XK{7XB6H(>!xnLjz_Gk)SumB%60J(aWWeckRVL9YjAgHIDuw-1&pyIQs_k&OCy z4feYmA9HobJPW2PckF&X4|G6e(jE%*&X(4>?NBbNp{;X~ib*j^ramR-+g7ov>O=IK zBJ9)r)A?du?i`20hkEs!&PS(WD|N`D*xB86f_0Dnh8&;0E2~XwEv3*H{=FXma`}tHZNxc&G=jmzgvGpYxb8QTDIoZeKIakPWM-Z4%`L^%BafHSJGDz zqV3iC1)f6t&-W}goOWh2F4YcYc6&aHCtYmOkuWm4J@OR!Suj_(a0X$F@it4j5$Af! zy}mg7W(L%z%$IUIMWp_nmm7C9isG{DIN_pVbQe5}`!(uo)^)bKmZSK0)_ft5E3Pht zSQgi?N^P*7-EydRRqIcTO<~?kaqSi9>%{kc4#JXr;qw@R8U2q#3y@)ol4Y#v1U)Rva;f{vKmy=d%%)j-3TFZFq7(!28j&H z#w^n*&99ZW&x0?BBi>(8a4!C+3c%PT^Vb@V8j1?S5L;^w!#B1@#vHEJPyp3pVTrm5 z13y|DI~v|`wYIWx5Ox*2_v;(Nz|WXpbKbl2>no0yV)rx@U)+(hwKu-Q$HB|Nbq|bx z=gu8b`!^=Sub#^MaX9cv?4FsUBUG4^)5XPw!-a>#*4~tpTS!QVlj||(+n1 zUq?RXD~&HFMjiOe??Q>z$OW z+NoAoj0xD%TBD2p{0#iTir^EytlOPW>@UinOk6c47|y=l2}@V%QGMI>>k!Pbuyj$J z-zNY*Z46&EM!d{gdDvu3@aIv8mkii{eR8t!Yuk87j1`mR-{&m>#8rHeT_a+}0RlF5ip87T*Xq9+0L=0`ZS! z^V1I~!F%5l=?D}~+m=vHJ@pS@MHMOan$0->%3eFro zTTSud4`wU=plgwlnf7S>tSbKp!->A5ydxh{s9{8$?} z^e8zP)lMMbsBC12dL_||C@XV{^n}MGO3T6qPD?{`AHnY{#plltQ!qL0x5ZfZhrGmR zv`dXd554vG+F$B0nRXh>hnHH6{JJS#4-<#jf`l8u|B1EpLW=2+twW@ zbX9?Jck9o3yAy?mcr8b}jV_KFE+=4R(!rD~L(1Z1;Jk>lTk>p%*5`g4RENeBVEUDJ z&Ab@}!&ij1YIf^awo3X6N>k{31P`G)dA0kUCMV0W6^duu6|*#3BnD7IBSwh6k%%Xw z_wg4oJ?@8Qlcm|qH*CsA)OZ9B`{xZV^UFrHOq$=}9Udbsa`U?%=H?lpjCliJ9APeomOCd=BQQ{yyctnZL3WrChL++zzaJEDKFPn8gStfR0m{q1?ZhH6K&f~tL^8Z&4%=TTv8@xyhqoXAHBIZN-f zoDlRbtG}oCc6JU zCzOBkn?*2tIdJZC)sEPaa?RbDI(DCipNF}VK+xtyM)^($oWsDUW(kE$YonG4*5%o! z2=NO;J|&Uu(ww@ZnVN^|};n-of+3b)!r!Qd4cHaL_>2#88UgV$=S<_K<3z}h7Psd zWF}~nCwh`;r=y#VeK>Y7rf-Xka4QmV+IU^ISx^J4(Nop_?#q!{H+0S=5Ll`}=z5O~ z%5&;QJ02cV%PSSAj?c=O>W_Sp!u8lP&}UM5ESdjYsd_XF!dAug?YZz(4?64W^{=WL zk-ga#=f^0@OCYt?nUkKW+Sim=WF`2(dv5H&pDbOBA(bw7(;-qL#l$393#bK>}pw&F=aX>DumC}1y_ zI1q%cJ-eh0}AE4$Vi@^}ST1r({Y{FE~ASr*u56*=XKE2+zSGbH*No3f5Jv z;Otl?Scb?-GO`RMB-E5pG2`ClJk>!A-~4$rJ9}SW#{aCk@?Hx~I+o5AJ@@s@Lkk9V zXq=_s@+XyUp9?gj|0B++**1zv;7-gcm_a_Y;zHku0=AXGCvS{lcTV<^NTo`~;77$f z(=|ws??7WHJlWL0v{+>NoDLK26g?PLVOclGqqL7{V-W!o{!NnSd{6%usqK*^MSb5 z|L*nkV64#0fcCg}%4tm3WN)sCpWld>hWGp2v9xX~Y9+yTa^O}yL36}$!OtH<%1GDa zI%ABpjM3NG&VdVWvS?vSW10{wZ)it&xj%-&~5Um`o?@#Pob_?Rs^Y#X9eFZ zueH@nHf^&w(je#@Lh7h*gFRhC7iFK>Z_QtQqQt*OT_hg$@?)J~V_T@l7QN3oDw>Q7 z<|K3oeHk7C-a>_FYq~<8va~QW6x!u2HaDw7YvZU&T3Af} zWXhs&`{vk1<$Ej@I$L6%bS{_hgc zqbUh0E)Mqj6IUcN-}#>b54s|(a-p&9zOYZ#z#Vq&a^4QlcXbE8NRmpvce$=7`xdlB zpes<6bSwL#DN2WiC9NbjI+jHk2#$n=9k*#gSu!d@K1uhU87e{$#K8Av&c1Y${!OoXJ1`ANS$B(u7 z>g}VY*~^OxFFGoETp0T>9uqmaDAzp_i{U%He9(|i7b0c(`DVf-a$({l>bCsi;&;}5 z@v*lJ%jztO)8PX@*$jTSS~D(lx?_EyVmvQ++tL(tg4OLamrXoH((a|@NPH}gCmQsg z^qLLsT!zP}rs+f#jNW;?3EUH*pcGF6K5lg8b#2D3PGdi*2A_nyyvhJ-{b}}Dz451> zj@SH0MQ9qc<*1`weP1QBUljJm?qbU2c~)I@s}6B}cx4LA9dg0scC*oY{)=C(+R{73 zGdmKUsv)*wbe37p^ zT8MEfsH1`mJ+|h{AHZb0+YD)C@LYvSKt3U!b_xr456!AiKCpw~c2o0l?eJJpQNz_K z=#~^2eeB#jkoEhQ9ROT14D*?amsVQ4(;s2bdj-wO>iQ6$38x#^1}Nmtp#$M-_xM+Y z_MYDJtaOz${UAgwIui`LIm_uO+pjt0bNTZyC~s+=5@|A*x(#)~+Wv^taa{-rP7p7L zZ~yA^AaWe|va6p-NU>S5RN;raIteTfnJJ?BigA>|XI!;nYI&Ono+qZ8dl0!EMbv#jAwr?MaPX#o_5`kpCl=EQ zT6j)!JYYQ8$3HCq@a-g2$0}ar01_t#39NFOOwCarR8H+H%F;;s8tFFgIF8fZ?Q-00-85B-Y9oe+{ICHMss%S>^jwh zxTt3Cz!TnMzqad7_TR-+ttNYHy~kxqu!H*%T-ow)Wcz-Ffa496?gUXOORKqY3DbecC&Z~_Ep+%r)bK@~l|4Q0 z6JwD`gbV(pU`#kCjJ>syor5m8@_LBR!`35WD&fX?l&%5kYLd$goXT{~cZs@93e?}^ zLXLL*u+33xJMO&6EZSa_KmV@znN=PndiH*u$^k?YelNU$P-@_rjhQ(w3QqKyP05Sv zmO5`yK~F45e7FzP)(tIXCh(cRs^#j%vH~oTF;#7V^u2kJKO+4Sf(( zm6;%ijB8#?d4P(#f>s?VEmLk>=AO3dD@{%LL{ChP7=Qf56$i`yMu0uP|dQstSyi9^5>dZYD zlMM6oo+5T%kZI?u@+S}*KQbl3rn67_kX|>5&iq={=q<$`hw)#SoonZo@FlFQx&}#= zSm|Ng8(){B0@DYaxL+y5mM6_mMpR|V8l7(D`Oq)zTJqkB{{!Rn10&P{BzAf*VPB>>54(GK1 zKF?|}BYGTIC9uu;^JZLdN2xaA%&4qn8YQU}s)x?3^okw;2G{5K&##riy^`=NzdWC( z4}AS>^i>2KZDkr49q6@HXaS2_gmAn@h#^uPloI9!JohhLa3f1cKBqWFCte|*0Bc3$ zyJj;ZOCxi~eFI*)C93pRFOx?y?U^PRTK|iw`vVl;j@Jww+w=#wKPd#rJ>&vuI>3|s z>3qRPld_kxvCi$BocKaS=R(tE-YTegw5;sOyaU}%M#PzBe?$;w zhXxFa9Z_UduCv(Q&n3M{HPUmN-n0ea;4cM3913u;9xeYW#n8gC!M zhU$=6-4^Agxr#g;v7-{@mesBnb1Pw;z!xXW10(H+JwCp)Ipi~T&)&abr|2m5dxYO& zCy5CuqqgBDlE(R?cY8=_*5dbw)Z7rgL1$< z&h28E{lG=ziMjWUMKY|qp)eNA6JTpE+U~kuj|sq*ZR6>_S3*Ef!h9t_d2AKJfc)dG z>4;>~gBicKkkBs#?(Bs8NEFo1&?|m~I$<=kZ zzB13tPhTMQ?Rj8NQoAJYyVDJ3DjX&hBka$wYGgu3B4AFD5_RAE(8R_t%Ch zvV|%!gar7X8+~dPfl)`Z@-3D=DKY8U5+UK*e_|&RCP*qjlBSK7gkvK=Z~6ljbxCcw zpdQf$WSTYm3s=NEao<#5NMmeu@_@y6n3c9Ze1!|&lYBT;9*O6`;06U-ooy^( zZiEO#&Y}iH|5xIJ5s556mzhPUMpuvI+*NNG4%RIok?Dv67R`WMh#!yFph*&?Cc-^w zV}0*JXT!Pqexf?)oZwM zxe3OP(@e(+ukmta4Lo!s5tN~+PnjKE`c7TW=hSatv>x!0{y9ilj1QB-KmxRWdz!7O zLy(GI8!JrldQFA914l53Cq7`YG65grAI3po z68O>0@Nt|Mi@?T$F%Ji*&*@|YE!BhOrDg+Zj|Z^=>@>^2&d?AdP``>LCSHD56r*(u z2%;Vv2++F}k$f~)nY%b{wN(8XETBhf=#`#FNx1g1s}XHih~ERJ2_VC7sUpwL4a)2w zNoohQDC?pys0Th15zy*_0W1UrldOMGCM{5YlK@Hy4c8BO1%&@$>yBwQ-Tw%GGO1V* z5`{QRdz7IkxcBpK)g)kWptaMVv=<{?i4aY_s+|v@wF) zrJS=tgL>^$m}2SXKi4vf?DU}Pgf{Pg>`Wsua+9+->bq2~|EYXO%avH5$OF;x#c?y2 ziHT_Hd8@5*m6M(L*@Bsun`7K!FwmlYm(>5R0JO3Ml}WgdHwH81L!;CdzuPzUUS$gH zO%g#~9?E9DRC>obepd6vLj8~IAH)JOsfobPkSL`9Q>p*TaRoNrpirdR&Ub7O_k;NH zG$y)GdiI_dwc ze3D{^`sjgt_*#rPhKTEkYQ?dWce9xr8@&XPWwb<^X8?N`*>R{6vScFlYIikqIDXa}QZv$KcDQ>?B1G6bND5$Mx zYr>)1g~w|=lg)Xf)O^J!2<*6eU%&iz(9XSJHVaB^tAYFN3h1ACyVk^&uY7j^1`O?HinN&M62 zpA7mrDo+!Wm%<%OaCS*>dyS&#a^OSVVdCdz@P1rMCmc0LQtS%h7Xrh0XDjM?@a>;J z^E6FNB6(aJmE;eZyb18eFPXQm=x(KtsOgS4t`%ho9o0H)cUVU$^24`9M9jxn6m#Ha zyZYYQ#oJW>%XYup;;{T2FVILA3}6H{B!@_S}X^P6;QSSp{K6IeC~nmv2FZQ4+KY=nKbqTRAp zZnp?7JzVNY(e#)LS7)OHS$GQh(w~~}IiDZnjb8j1>Yx-^T2{{w?Y)XiSyWhu`Z~>T zIC$Su`E(6p*|5Ly0`1ucc1HXRPxSTkIw^DGKF>$0Q&vkG1}a{Lte@@N913?N3YZ_t%#!*fu^q{*gaV861U2%({erHOSAJ zqIlA5b7rt5XxSiU^{y-n1QPd@b3bxxB6Kw`nm%+ZCJ7%}%Sd_|5Jqim?Rb^7F7oHT z)@yenT0Xqo)jFF#LTH?IaW<4YUtYmxr#yI?b!BVMB_}Lw z0Mf2mWA8~wXN0`^)HB=H%i~x?&9r+QHF^QG;CwOYT-V|8lZ4l>ungUoOUK7=c2L}c z6C`F3I{CHM)U$S4Tg|ONMm0u8MxXucGJT~(&T(?zVf>L4O%r*z~nQIN0tjO1RqqnyognaY_ z;Q3++21b=Fd0UmD$%>de~Z}kTuQDB>X0hd}|$~L9RT1?)KP#Xr|T}1bU8nN?HYOEF@ zJNyrBzIlsQSzB)P3d78^RmxuOE*ycsDEuh2n5z;Iid1=2P-(P$NWK6C7|<_r4{Qf@ zs10OjV5u7e-0|Pv-tXKrsnB(Kt5XVG7v+z#+QyTC*E=rfNpt~1N??1#^&4j7svf!o>25*kqc!lY_a4) z3T?LoX0ji{u049}cI&8IbwYTA-B$^_BCFpVGd?_w=q7cchFnr^)tojgtzE(cA24>gc6JKd#D{>NbOZHIe<62;#VLN{m-nJWX-?A?XU9y@T4<&Te*!x94h*8qVBfUt#k zodf~pXMb%Jv!*)tgU7(smN7#iODSoK#z&B`T*{CaP_Jl`Cbo08V+_w?6GW?UtNNvm-VkBUSX@vD?+QYB^o)l!^8Y z2od#-Zl1KlMgaL8Cz^P(dW}~}6!p|!DP6n5#`&I3`ew_qrw-@e!S-MAyZS>#SyJnz z{Jgl(J#o;jGPTTOPvq4>Hy!Vj<*%|~3}|by{renDp)4J?*3-^N>u4pWFtrX%S8c3@ zF^U>;@PgoRqt~MfhJC`3eQ5uO|JaX!k3{s{Za=mrO6-vvMJ>cU$5pq^FL5ukc=uQn zd5|vMDm+OClns~~ zd#>rZTU@kp`t^^^f69W=8)ilrkt@j)91CJ1>2?cY6ra&?nRYViPn9r959PmF_DTmP zIDz1Y0`%h-k@xji!Va5C8>$3Gn01#~O#l^v|GUui7d;*l+$^l%7zC=yrRG7+uE%$) z2@?nHY*8_r{|;)}hvm8cjHs4f?Iib&>_bp%0-vSHWVy9SU%jKyaY$qqF%3*P!Od!{ zy{T}_D8ny>yI~O~VsV}#qw9DRTe<}`50HX@^d>V$+5{MjDWK3~tFU&TZk8plHs6@M zD?iUfd_K(M{(OhM#Oif5D)T59e|iJ6APU%j=f^h1x$|&t*KVM zDe2C}abR8mZ%uEiv|BnPpV7Ai1me*(UY$glu|U*Gu_F+~oMFTcG_M3+Z@mz{i(Cm^ zq9794DPn&WtcIULCw>-EFz`?XpHqX~ch=Y)IC#qwH<|ZT&`q|j(r#ooAa2(3K9v?S3a4;z4_XL^9j4JwxH_)8|68OZ{}v@ z9daGR1&|uC;GA#qkj71CGp3vtFulmrzOQ?v2&pg*oiGiVS$~3;ShWje;t%TYkUP@h zGI~ORN=YDD*D1{FNnlkve>xDw>e&i!I^9%b+1ppRvpk#+C){pLjs(W{r# zFtUf`GU{Zx&wh(~xlw<}jd2B@KDud~jZ2H(>0ffD7bwgTpz1 z5aa19W=EZ`+7z;d-T{{z<90a1;UQDwK_%F#vfXY(_cF^`rOcPOtIm=yc z8*lS!8;?vwiy=9{UG!y=rCiDSKp}YBevoUAfFr-|O^sP4<{2u{AgF@ThT1LJk675n zyRRGBaG5*+_FuNmdD5_^d%&Ofqqf^2oQUmgeAHkcfC2pcNut(Y=2MdaA&=i*-#=XLQTkF}tmrqhzzq;!o@bd&JvvQDFX>vgwKP08h82YrXtW_Wpg{oS};w3l3^x zx=B>}G**)fs$Y^}*jmP#m0DPrZ@d85@HF2MXedMi8k`pIpQBE4ugB9|Q;-j*fQvO4uhW1>)|52FUL9;BKN|s`-4!k2eL0cZ9I|cl=JPY*4e6nt+rR(&QD(MRj=_2vJCZ*q!?crW9J~gU_$~4`!|82*cyk#&8cZq(4;3B zvX^ed>gGNrN~g~@YWPP~|4m{@0P)*LT=#sC2>H9Vj+x&0yY5_NuUP5;k>GOwN8XII z&ZPge@s(zOA11g{cWLNwA(UmRce1qZH}DD&(EK}E@G=3Za=-L_0Mo&>|DPK#AGadQ zpeQ2;3(eRkm8GkFBj*I1gMjgma)r|MJjgUnyhApn2k2?CiUoz%-n(b z@gn0Z`X5O^$fA{P02+h@qz7dV;05``)ZCG+SPF^`^pE?GMr2{6oP2)U0O)h{h@$z9 zkjqi)$AR5$0XAk2fVoN2()=%Q;WjDR65)RJk*g+z79R;_b# zX+)x+*msk%KsJVm;|~S>Z!;J3#LG?htC&vFp!z-~?uoKChcWH&5%n6fqHqGR!t`w9 z7XI>^`5#4*FoJ7m5R@|~D8S&gGFY`;j_hWym@=x0)v*s+pRRG?NIET7?9g{@(3k6D zEF1E51#VrBep0mn)(?r1pTKd0`7wyO$F&fPbZ-F~s>%28yC#jkS4{kg__+W(Rp4zA zXRA6`?%8g`Z2g-D%6g2Rdw#dY+)G0*N40G~cSM249>t0ZeM)?PCgKE8YUy1;;I7L6 zI3)|@he4B#-$aRN0=iFp7K8X#(tN6tqVvgU$EAw#N)^OQa4>*Qv&}H3T1A?zl9RRKWPB zpi&O)w@t@|BFsBDZ|7C5#DDdCRMPO(E+j8W}NQ(hb z(*u?IZ-mp+@rJJ3l;!YHFRMU|@$f5g{5ik_e?cdXziLj~5emTw+RfMRsh5f2R*dI` z*8;}Q@hBeQFt5W2M6zg~&?=q3TcByOTf0$=mc;`!An+qyydg`2bsjGMuvI|xf2=Hu zdRK*Nr+oy{4Pos!G$<_^QPch`&sjI@{QNtf1I--9ldDE0R7Jp*mF%8Vb-WPV1DHv1 zhtB6oe2Qq+GJKP`4BytZmF;&vLT>96PFlf?udiHWhvMF_9Xkaot}=*(g(M5mJ~iMQ z;a|^oJkonsA}(Xo_h5?v?aTALjsDT;COBU8A9Y|QC{(c6m-GE%NvJ-IyKxBw34D^;e9B|Dj$bT+(-&t_jg0zCw(tFPg8L3{s=SCvO^Xax`9fCe58;0m*UO^QE|Xf$=vAYY7A z&I0>xyNV6Hh4A#(zBuXgiRXs2j|ZWbWA$fXH=p9BUr#KO#~x?%p|r0leQJSO%!B2QJ@I{Ut_vG6%%|x*kCjF3R`IL^{b_g{Mxp+pJX}*0b$HX5Qi$P>s%f^sk;tcXm35xb(uC6 zcG)#LS`p?OEHT)MIxmiC6^=rvWyRBR!v_P{7HoJ|B@j!*i@p{Cbjz9Rp&& zgtujr$Si!s87({6UjosoYkSfKTN5GL=1L0dEQS;p%^Y<@zA7~;Ckgk6j+KN7U5uw2 zswo*orptit%PgGQ18}BQY4J%4ASM~Me3DjhvHOy0J{v1;60#NVG=|>lEr&4G;Mm((( zj(jfv?Vk+}s4IVVdwq;LVzIHl64-}n@l?1zG1~xd1_oe23VtyP9K@U8uk`((3usHu zmNgE`7R}q4nP~u@{O57jv?x3xm~Bpep04EY<8GoXFmFj)TH~(AaURF-ByU7elHEp!$ zFT9mTDOlbWF<_|;$Vmf;l&AlOkg=2>wqrC2PzqFGOE~)Ux5cAT5_z@m^hcQ9QS+)G z0LSHwud??oy`& zebxa|Z7!=Cz*zn-@lugqhyMJ6(hB9ri9rFbA*d5TJp4uXezQXNuXaY4XvPg4Kv0^r z$W>D!+@YfANG;;*!iaCMML^{2hWwo+_K?x;-jD^0C3{l-=){w^|HK(E{??Gz`w#$= zW`YGu$>)QD1sq^(b!~7#uTe0k4dv%x7omQMU#cb0yJT7h03(7d88Fba+#tsv%IzaC zx%3i|E{@2Q*xYXmpER}e?Fx>leTuy~>A6EirSU0my!nb+X&cB{7nsU!d|h9f z|D9WZ`jG}o6}`~%U2P*oBm8hD48a{Ej!yZiIK*WG4l&f;Z;J;W%`cOc_t-Qvw`y*R z=MGgqfWN^j{&pCCAN%{=90Cslx~TJWg{@b|`8hLQX9)oF>KpC)zvKP4BAOL26n^p^ z@D<1o=>ZO+*x>)1@+xjc6CC3<0@XapWol+-Fv5`XKS%Drw&lNE8R+!if{1wn#@zF3 z60Ja!g`_`zR2-GOxl-m`^>c^7bhaRoYrygJwwtwRPZu>2Lxfu65&hANg+AsIT zQno)34dj3@;Q8Bf!SRJlF`n&uHjpnJs8#nvKHRObmtqlHJ6J!8tkRvv|9IDD5Xti| z2TO31slLa^*M#Bxmv9(b2`{VI9=vFA4q?M-*TkIbSrQZ*sPk!H?icV@bN78$xbG{5~F&Y$u%nFGi64&CE7AKVTnOo{X) z9<{Yq)}OWq#1tm)1(&lp$HUCNJFBaEhS+WMhUBJqa1yEL(}&ss8|D6WeV8ihYcRr& z6A%Y8D1k6Wz!&hlkp7wv+4;KuZsYo^Vm?QQ(g_Xb!P{v}P)h(b5n8(GKlbIc-e4~M z*yu`FM1Ug=fOoTTM0_wD*oYfM;19zl&@&J2#V$-c0Bj?$wm&BJf4P>(;ce;quz7F5uC>(V=b>4On*upGpm_(E6*SDh+_6tK=)3OWH2 z53prt`yRmiW<3oT2b@DSF1Bdk?GNOUuWd;Wzf%sjk&f&9`|=^zEjVK948eDDJlUyk zJC-$K+>3d^3=QxCbr}XXCmmhS72#t{>jmy@QGl$!T<*q#E9*f2& zT04Rd#O?ne=3mH9V;3{jZ*$so~O#|tVM+U-(I|dhKR|vBi z-Pi7-`+P2jer$|%4Oa|N3)m#Ooud$;wo2*nvJZ|7u1~3%!iv#mRvu;-J*yv-aAlS2 z0s1hpdWPwLn3FitxGqsT(A*U!I&XM4l^oBgg_VSBqrDBhh_U<~;yrGNmV;U5Pq4Xv z4Q5ex(;*f6NQQMOUE>_QGPc<~{wf4q4@CjBOVBIhr2duA^V1%@hYXK8-< zm^KWQ78im!`7yGA+d>e?!jT4Ofg??(}Q0C0l=aAmlm2huKapJ7$gRv0F$ep*6*cFR#4Q*rt z?_UDV@M}~%=`=9Mo~k`rNm$9wO3pzc)quPI?PtXd8C}apRDL7L`13)ogOP!?uOkxs zvD-Er3%Q(taOx?(7g-9JkVz3|>Ec}UM^-6r??IspYRO(!UBua;=}o*DHs1j9yks8C zOAL&Vl5hYA>M))@e+X~Cxq(uP0BONMNAsSuqK5^SX0;ikBLlTtDmwg<%aj3GKmMOy z+&?V4=7|qfz{)n41@AH441@?1Ug$TAe(;{wmH?U+0aCeb9>O3dTsHJQ9T_KB+2f0J zMcUNlC+!|zobA%8xEphzB?v^pN2=Z2D(nD?KrPEk*lHqQD7ve(7ZI`l^RE6b+su^S zkMNaA4DU^Oxj%eFS-!}vdLpzh&r#Ki;px#xl}LioTk@X2Ui8UrIT!};I`E2X$o|>v z^~HWfSaCGrxUN0zWO@G5vY@m8BV zg`XSO05AJ(1j}_5UIurFW3dcf;l{{w?9)4sO+SeXy(;9CnY`kU{hp+q4I^i__mDWy z44UWF9ZrVlkk1K2emJLXA*G#V`V$)d(7VQnXUXbrm%MM6TWVn2u=vnE^i~sb3!M%9D8>eDrv4a8oSf z?mhFNuc1JM+7ila9)~#`Wi-OaG4`{68Udx=EsS@r*}ht32FxVVH#-m$*t25CFKgIp zo)NCz`rA;V5%97_(IqE0KlA5~cQ<

(*o#iIClOwS|#XDDbh?hN$*YCG?!sFrox z3kWC(s6>e>O5;|Nq$aBfC{aWSl4+EjGc+I|SsE3QoJ2qbnw*2=mYidg!A3!Hkj$$E zy2H8WoqOMS@BOt0Zbmn2t*TmIeRIxVg>rrbZIL4xYzsbB*EH+LV99;6!;2WdGte@( zfo5%ni!pPwb|uf;`p*RC-!#!Po<@f2sANzWDZxDbia1;FNAS0ltoL-&?de@If(|>X zDNZV&?FHDClj5&p*oK@?o6WmN!DkxjA8vJWO)upD6W%^4P(l zVYWYbJ}R}2F$5iazo&ttZSe4<`c?oUxJ}^pXiMIS;_9I5Xn03Qz)bog7*PxR5+R~b zNb!kbB=+<`X-F)hDVeSEH^VX-YEH`N?>bssS7WjayvjS1~TEP%^OKlo4?44*6v=8cDgdC*LM zNGQal>W?77zGxCt7Pm44u)jv&-;BE+S|F`Lgutt}yE@qs!UB@lcE-2Gb5zP7| z1gA22Ex}XYbz{+gtjqD$B6Tq1MSur*thjbshTFWZL%e7{sB7kZlT5M-OY7y&8ug6W zuK}(~@WbW0P+X0Fxy-oJqDN=1$Rmcd?=wP*T8h!=Pw`D*lKzYZpEaI_hvRVt_uw;F z=ENf!w$+_6Ha{yVqQaC!8~gnzF2jMd#SH!ryD{i3B330xTcRsP9N9%sGt3-8xEXbp z)_n?b(?bT$&8SS!X!PV07d;4QpN{-Mc}JYne+?mGCDvmXK%$gYOU_raB@cvJ6}UZaj7RCr^H(I#e)~xW zQiCm=*RUilTr3+i)6AW3fQwXmAK(juyS~ZB^sC8GUMKT08qHOpnYkhbhipT+xsAIN z21_C{7Z#=g9t(_AwfJ9kQX7BGpIu$`{~ZN zs?D94XR~od{&yk2iVe;r?GZVN5PW193bFjhxVkY=N5MrUH3CNs$Kx*Bnc|Oni<-V-8y*3%J83$nK5hiO_0zs;z9?_W>%4LPMPs5 z9lTy|{e4t_sf?E#60f@ENy{JK8BmZbvY(lq`lU;Paq{k46TSL1`UK-ss{QZ1_;&}% zeY-%m7KvdEvL4}Nvv4pF3v)f`%c~9bk9u}-;9@Wa=q(q6jN<*y6%xf83Anl#<-mMMB?y|>n zjF8)iiP!^|bVL7%Uif3x++0F31;$YBF#l|G%+1Pd416&TNTFd0bNuKxJO(pw za7J1tg6%YrM=2Sm>WtxWZ*n|_(+KTWKp)T@5&-I@9k`xo2outyDAI=fslC6+paXLobD51& z5E!a&tPM~5;W^?eJo$pMm59MyZ+ic`l4oAATn-y|5JD*tzXl0QN=bTcS^^vJBHxP}vFf05V8VE<(ob za&37Am?YLdn#?76m3yl#7Q|lU1jSJh9>$G=xj1ckYzcrE2Fk|cJ$6j&fN4(cUHM{J zOBD-)OQJm}At&y7Q;tBMRJ%!E+GwB@Hk;~L*AQ+piz^}negf)?CSwUm*`nhlLDxaE z6%uW&9XN`w9&8qW9ulTtgm*%s94`IV8NSVm@z~#q0vXvM`?=nx z37RXLWQy%@;8DjgJC5rBiF@oktnENd$Om>-7TgEnBFWr0)pF_>;xb-NCW<7P=XJU zhC|}wm^c&AwEG%Xat;r6vFG}&Ja;uk)O^TKvT_A#qCG1&9t{*ZZ-)f3oB23GI0;*bIIhfBZ=39O3aXr zfwO>hl5ew248t2Csu9?-xpNl-xiN>7iP1N03STPMLt-eb9Kbvr%9&ud<-J*n?Jq|n zG4}`w?XH!?5ol|jo;GD9T!MLm$JPoW(weMF>?Vg?*ndLmUz~4Jvqs*c_Y7(pUuZX{H(g;(2QGo_GNqV@2r#1g@>^nvc?|-Tvb2uydY* zJt%g}Hs?A~8B`;j71Je$HH)}CJH>1*mwO@B6Q#Zu=Eo9XQgGzrLru+7{o@v5CeyNS9}#$UtLim*xkVfc!R53OyVfZ@5|IOAPb2AiW?Rma3uq3;0a%$s*Ew>< zNYG$TUD)eN*Rg4!-uw_qPFS4LsjN4i$SX5@iWw*JqJyWQ@j%)wx$c!W)}8L!De1kO z6>i^=14;?DQJ(HP$8Y+d^$h7z(S=dQB)3Ym`^Be1S{P@M5+n{q=j^Uo2mI=BQR>nTKq&=oOaBn zz;g7atQ3v3^2lm+ay0yT&IB*+s`b@S?=E6Aj;H$;9+6sP`c}o)GrFk>JN4dlEa97t&jx2P&R{A#c zx98SV^+r@Pmk76kO|_@pC(*9HiFOOnN>nd~IEEK+MJpGPAu>hhD zh3q}m?X`a6g}_ffJ8~e2N4$G3xkQxE)dE|@PpfVJy6+s>x1j?)-9xj z#V-=WiYEj}(1|v-GR$;RdN=pLn=+pXPm%o#b#X3tb2a6iF9fWoV*|KNLcNtqakoC` zDV``E^BQ;ofghyWfutE%ub`o;Sj@1%Z^aGLMX@lVKwP!ghv zAWB*GvlXcZ6`>O1u|G<)e8SctJp=n%@9uL-eBsx2?LrvEVI2T_azqyC{0-p_U( zTAxxTs?t%f-EZF>S(l$&V_cKFjj^`6*%Y!(cCBPTpcKF0oMnrNY|D&e$LGU+E0J+u z*hIYZ)M*i`=fNjOHfu+SaI&j!Rg}c~CJyg&&sFC>-=>=h=_t8a!y()HI&q(e{_X+X z@n9p{bEutw>D2N!zE?03+`XJbEGTxYRFabGK9ce@KRl#?@ArnyalJ`ej+v4DI-rqz z#yX&FK*v+=DXNZ)qs|({ItKs6N2vrCj|&w)sIQ0|?StNemF! zD=EV~7GtAV=4$oPd+UY0WGh0#I3nu|+`O3-p)*sP=MAQc#fiD0N|(H3T20d8BQI#eJFD-J?n<*Y zNYO5!1f<-Rt-Tu!Zwgagh?+3DCaUkN`Fzj{&Vx9dYGxW4t8~M@zKo-`zJ0MFB($jR zKpGiH#wZ=Xm2k0M^0u2KMcf5pOZ(Msoi8l3Fpe-qJf@R4HS5eImFnDp*EcvtmtRoZ z$`E^S*M9f^;CFi-yBB;BiJmL0XmojyKvAo~8IMn*2PvKU#y`h^n9F1O9 z_qRCeg$TFFqx`?VDXj%o}>bbgztE_fOHK~*kJ*# zMTVrYXi`FI6e;-&IW!pC-f!s$N$EpGX5>D?b8A(91`2?m-Syu;11LjS)zf1&NMc8BoKKJ=s(?Hr3ig&$=%p=l0z8g@(9`H+! z0Td0OX=ASOTt@$82L=R*ptuk4o4m7_LJ7Q*1daD$j%YPB)vM=Ji#9_bE&zE1UvL_f z1sBMT37a^RDR-3hURU+#7O-@)yHVsj-SU}Jn0|EF;QrpW@{tPsuE=X#`-8S&n`n`g z%rZUXQwIJqpbrAEXr`wlM$w=7R3Oy8W_9#^U-$^GmJ;S}xLEb`_O7pWw))J8b1c|< z9BnYhnjA8T-9hwq>9AlnmJk4<$$z%LgT4PHO`Pzs)9+CEj(B0A?vttj@IBh{(Wn+Y zaVv3pCfejCf4g)g4_+e=3RewLj^!0tf@_7ow(lXq#PYf1)^|`2E(XnHCWHBs-0nHxxIkNZh%Dr2uhzx%?NU5J3(VFz zx*O~o*q9%G%&HyJBi$U8 zj0x+At|RZ-$G>5^xH)3Xel%AtZw#6I<-VCVSRNYCDg2&l!&k;B-6H_Ss$M~Mi^BL= z7Hl^;8og!O0#m?zt?#!%n5^P8etpr*#ZiP!HK7w^Y0Loxud*ow)33)XX}(SU6q@9r zaPe*XJh7zvDJc*Cy(j-TnE0=-q3EL1zFqbGzv@@%& zSd91Ngw;qL7Vz~C3r~!ddkM0?ds<`aRmD|7S9u3~^YZ$uGt-6({cnI=T=H+^;?r5! zEKR}PCqrHY#FvpF5avStA5Aw((&Vmi;fWCU0d1!^;8|=D-MxLJHr?}3mxDM+CSZ-n zriBrQ`BVO}*)C#94Eb76(?udn{IhZHPjD7#1^SU#`dnaQ4BoNXJF=vKhykXr!FU>A zJqxXz-2h5;LFDu3!q$xxfjB;SX7U2ijV1ygC|k_V)T>NW|MpGVA~3XpJ{`m{n$$W5 z6G2fX?Z{3}ohC{X^iu4BSU%SK+GnJ>5ga;9znwVY-^+ zVjql9zHEe;9=^W-Ffb09F_@U+5A}QyX*-eyt9D(xKiuCGn>1nUZR2SHP@8p6TfVAwZr(=Z?ODy`qYUA zldJqkL0xT`M2qXMHgcC{B5*uOhbjYOmPPSUo~>LmkAkp?ZOxdT)=}mJvMZh~`%?2A&FLWe6Go z>PPm;CY27p_CSn-^ils0<4?(^ie{r%Q9?Hnx_JYr%8}Or1c|;~CU&|_{tzxvPu~i!l@-GYCbDJef~8%1RNAmBf9j<~Hrgv~CBLUq+9mtf zKIIpQuYon{&*_*v35jzCARGFSSZ|cjj78IbMsSVp`qCm-C}_WaPhT5c74|_Myzirn4OlDFm`EYtxU1=EOE(z2c}mA&?SBBm zOc?is7eUYtincDJ2oC$$#Y!?q|BWVd3>HgrNSfvRa+?HWzyOE1pSwrwFUW=m`Fj4- zoD9*ORi6+h-$OMYwEDU%KTD&9w4@Z;YZ0Tp>fDr@8 zE8(`eNSnc#VtGs;%jqnxrZ>xEt|WspEu%%6YJN!}CZ|4vq2}}dWT+L7I7SG~ucMD& zfnb(aW}TtP&oh7Z_fV=~L#?C6D-l_mG1m9U_(Ayj{qtHxm>h494nw_6&wD9S(2LT`2LT#&2G66ZgyItaCq#}v zw>;jS@i;FPNzpiT!4pcH{=hz{;fjIC-4~~FD z$FofUmFW=sSrF%E-27xy#X{J-sm|oyaj{|YnXes_fHdvbDEIvl%{~wGuXY)SPg-p= z-^16eJ8gkW0bJg%boJbO(oARuW!4E;|2^5-5_BqWiFFxn+I((vY<&7s4Z}7x zMjw-OGr|CJ5p2|;xc{iuvd~=yImjdk;r8wrjO=yWk8aA=^#YHRje}%ih_n%|HR>5? ze`jonyiHZn6gzU-ypQYG$SvTzJScJ1PXiDa646UceDb=|47yG*05sOLdVZJBN|;KPmg`!7RkbN_t)TBgmkz_&0%YgNQJF3HnQwFlm)7+k{N;`` zM$NOjbfPw8YW|MThrPElgUP1kT9%H^Jm4%dF1+{QGk8}@qg=CZPIHk>^oO{c2<&BP z78WdRTJvaxgs6+3F*~6a!)1|X0!KT#+nWaa(z(n>$rPgl$f6zK1)G5zQ58s&RU;Lg2@9={I{Oa< zgt-xDi*DFAfmC-^a*@IK&JR2pF;fE>YkUkI5k4Jxr~G-uv9op#?qH{lgA|uv2o86L z)gy}*B?;wq5C&KA|F!?^FHm5A$yNlGTw*$n#K2lm3By4*?gM1A&y>!T0OedsC`{}_ z<|v?dw)u@~oAy)}@i9%VI35e*P{9O(rhmh5sMMK!*}DTUWDr{vAS9t3{9mxhRCHN& z3%>g`hdw;3=@5alt_2RD0%dXnsQOKJNvwflUxh|xxE1rzevr(0(Ol>y*A^qaEs zyQqe;tqxBfIII?!Fr!lU`REkDAPBfh%BKuaksORJ7QWPh>uJjvn+%U@>uI@{`8BU; z0`?z(ee$)_IB)j}zK5xT?o6)=DtknrnfDAO>dd_1LeX12Q-}*}%?}I~Oe)8A;?-ct zeX!bfv@rjtB!2H>T4#+!t`xNJMCEorcaLKPlhozrGO%q~oAKo8H!qT2)Q>FiJ~N-^ zXZ3%3g|R-H&GMc$GdG~p{yffAf_ zwu+dAAI3Z|WLm$Y0^GeAF)ZejTzdK0;%bo0wS*#UYpuK;tvw_@#1utX;43=Y58xt&{~*(63@OddOjRE-*&4j(hS;j$*&1-04&SkR>Hj`xmevV8XH_v;*l- z51^tQdCdp8p#xF|2AzQ2E*Z)&a}Bc~T8z>4&5qc_HY!+h^X zCiq^254j;Y?V`zjtpAig$kmGUeD?Ir2QkyW66$3zyCap8r7vU(jw9D*oH}TVw}5;* zG`{WS2@X?%kRUe!XXpJkD8Lr*f81=jfyfOwEh>Dy;=UCSP6dKFRQJPofKQ?ooUmX~ z+z~H^#N?hokb033+G2!d)U9ZFG~&n^R!S2Q@F zVbp$cWC#Lxrn>!H+!TNuCPp%M>By4ws9s=AOG4Ou=NpbVD!(evBZP$dhP}gz22HPS zQ1JaV9wIwN+IXkb>f}t)gz0JuO4*JlK-6w#$G$(X@Cw_3d;%imCQ9~~m-S<|+C|so zp;|N&@-;xw5l4a9SUc1xifeLfWx5RTg362@SfYWW7wZM5T)Nw#*Z!^^bVi2xVlA^G<<%&Os#d8EW3R2#LZM8V; zhx+(dk_~7UyB08|U?Q(S0j{*AeF{vbQhMd9;uAP-Hic+ID~+~Dp1KN0<(t5<`$h=Aj(NQyIWBmroEM zJ;Nm2>nI&3?2rfZ+{`vuy783sg}GOf6*;en`DcZDO)dpsx6v*(e~7#P-u^Ae**rpPvpFyo3YjpkWtcT2 zT~oqU^I+>BF;KkeNG)LS!h4(K`^67-A8%g)%A$N^N32kO_w9+I0muR)bMa(xGWpAU z%l5Rw-wXk^E()p7xHwp1QkJaS95 zeJI&p9L5d9(}e)`$%R|Ug~EYo$ezIn68)Q3P%c4GW?{zG|8gCAuL0U^~S_0;8Y~)c!w*V0z-zB^H`I#xF3}2Q~pPX3r!p| zhX1vuc%TGTCJr5WzAE**a1zhit+mRX=>5&17J(q}K1v253=^s$loQGdFm=9WAiB8k zH<@zP+}jNd0nWjro_u{a2iR0zNiEX4qFc-{M-LslM8w3%HoJ0o@!{O3=@u!ZCDZSy zlo!-I=Y}cJdUW;Al`(cX0H7>LMxyKAAbWn?Xy3KdWn!3$o2GE`tYx%YpFDK|#JHvg zJ;zZ+H}h0JKzJ0uoPTcPh67VT%HgbJQq z3sSvLV{zy0DXF`%d7%8|3ZS|+Ou#+3HF49QN6#m>)QZ?{$rgeBnpb(F#bh}(Bg--R zeS&VA-OqfRo%_>KpVF5yJ}vcC^l!CQbh#*W#2~5a0w?uc=_%5l(MwvWD6z3LD98BZ zQX@p1G&C+uI0@m}Vu)Q_T-2XoXs*%{`N?onw4D3axWaZ?>;TPAxc)KfGW#M}s&L_kU zFdx?1)N?Vm|NNxXIZYKQ;aMemE#cj}{%O-S!VOMBoeE=zu?Ig$F8ckMOv2-$B;>w!qFtOzpH-LnM;VZf zd*sE+G}XwE`wvo?p*dUib$?&0y@Uur!n~HGc*S?}+gd(VrUo<{15wJM3(wY6hnkOhlCnCiKiVnwqGuvXL~ZDpiKVf>M>a8Ta8lvEY_* zN0eQHS@L;CW^uc1GJkU?B|dnraCoqv!mYh;S21}-tIBqx(20m0T_t-xH=V1saoG6f z;`C}XHycZvlm)qTXbpyqLwK63*4lZ44kZw)#mF4P!)S4SA*^4k*<;%T>s;2@EdRahxH@hy|b#9;_^qRM->0`+zjKidvUJ* zpl!wTV7><>hq7n0>{L{V%w8HHthg&++3X*BPR2s<3~}<>i@r2d9xx`N6NVXIGW&!o zJh5+Ncrm#=2mT6pDO^+l0Uy3H49f}~mL-l;+ZN7%1;}7Za9wsbb2W5Kh>WVlk+Rg= zJg#D3gniNhzrjvXC`4CrV)o0qC4-M6+@40Fw>K^kaeZB~4c^y%OgSENk;3?sNuO3^ zh>dIOAW!lo_aAhp)21FW_ck#^jHjUUBLI7i7p zDRneV=3Yo1psmOD?Z7LF3T)cyx`G?O6=m(fqlp^6ZFJI^X1K1zPt}@}H&XeC7pA*< zo3A6}xBc33mIW6+^Q9YDjAN>-Z)|P&GrgZT$Y=y#Djgd_*Q`Iq|IcBMc7|p%Wr`Ac zs>pnPIF^P&h6@^T9MF=0eQ`8N7|*~OskvZ3Lt=`#Tffun)`F5Zb|oSF=Wsdkk5}c< zbEXeoa>Osrd|j_j@1UW8(SnhiP7@Y7^|7sHFvuo6rmp4tq{Yl+uRP*oTUVp%8JHl9 z)*mJ-e-;-e!PS@-YiW6S?isuL(t!_Awv!8C1-=em_H#1AopqPBoBIOJ4yeI#8|3nV zlE^_J)z<|rsxt!eHu^=U5vvX`iZ+XkD)x#ZTkKR|Hvw@^+{RF|MX-PJUw;F%Zhv)@+#XYUen%tH(J0=!PpGEwR+*p;(3J@ z&;7CLWp0yqKkT?p&V*Q%nmk<_Y1`!r?(7wM%111?MF$p4rTZ>Q z6rP@{d}E~c2OMF{QDkf9EL1u*Gdc5~r&nOZa8;QRj8;t^H1zLc6(ORr(I#gKg@y-% zCTDOnQ_FV)PuIqZ{iJS}|ZkxTzmJ&);Bd4gOB+fo4(UdpS3T>B*7g2 zFPVPgsTY{sIDhF~=UcU0$jZnc7zuua3^|Bt0?<@v( z(O$14oI6iOsod0SDKZ_@Q6Uym$xpHxzp}L+j9+5A5xuv!(jXU+WUf%Ucb-&7ywY`7 z@#B~>sRsT)wywk$@;gb1^~%keuZsS15!tuCe{wLO!)r_l4Uc_{HM_`)xOpX*<>iwn z4^GZjiBfF4@J>9Lc3|h!B*@mO^j?|nPI4Sz9(+5ezhluRO)fN)G@pEmh9fTFN zGtF=MI(Obvv&qvDmpd-`a6HCdwX~KcxuL0i2E(A(L@F>=w$D#Ro$U=%O1O`9-=(O* zWj%hyX{w~4;QLkT>H}R*d|Ye0f4S3AO;GsL<7^+e~nj~_>fk{gP}db%Gz zZD*A^WOZIkN>9H)LLxUYSsZ#1v3doLoS`!xsi2k)U_o9GwmS-45>GrC1ys$w^e++Naan zW4$!CU98voE!h9iAnZsGi&)Fm+`FUswcqp}?OL?3GxrVJ-I4=mpI!z}qtpkjm}!jC z@a%MAWGDyrL@jIb`x0CH+Hd*?3r3Wl+yANp>4@!FAVg zt=>4?GogHranZMHUc7#5JX)J8$49bTc+EO4WV} z;oOg`$yo&C7kk8Z$(siNRn9=1gk*BeJvLN+B|jyt{%PfPy_|2Pv&l$z-&l&?$)PnP zTM2_um3-dG-tYN2miugUaJ|Hj+uA}b4hHXR8=TJS1P!pa6Px<4d5tSnZ{Jf?e3u;_ z)ci;FJ}+X2uk`FBPtwYcLu5Sf;~yQt24e$tBJSg_%O=f`n0DbW?d{$lK8UYwCE+BQ z&F(I_?_6oxt{(b0b_vJIOF$H$<*Zd^y%H3;WMZ2ts}_t1QJ+~+_hE5s*sdOUwwwFx zDJC~HcSRbLOJwwZTAfvAZ2=Q~nTG{DpP5<((f+q+zu6AmYPTtMpEWOVfW!CicQ8~g zXQ~loRm>}ezP!ti7|qxeVo$iOsI#`4yrj;Lix#F+r3$a*Ss6|cbb1%-qULFEh%3pE z7R?65`ULedNqzFIABUfU(l`vz)gP1ThUCtVfTA1{ry&1AgnM5xhnaq9EVu9 zj(rU-FXk&@-r7*FFuB&p$9(Rbqy?xjBspJMIqQo`g^G{d%e~Uq6r?G%XzJ5usL{nGy!{ zmxaX%2UBOhX4a6emniyzZj?zSrVx*1td`K40wCCVL-D%=n zAQJM%nSL?qhv6;Fya?3}#w!i{`bqz;>jd@2N^ z5|DG}0~f_phXOxQFe+7_^|aO?G^yd7Cl*Lnk)oic5KJ$ox{9L?sz!kSighH}_8pU$ zML3(LlEj6mteOdvZ8>r{>%bsu02)cF!wM0RI!62!!6k96^=cyS`B4cv>gLo>Hp%WI z);mANHi|P0nsROtwjhZxk=>5aEx|g18zG6@`s%7}2QerwyfQtKY%VmdR_CMH6+DEY z)cjKwlKaxiphcVn+AhS0Wc%o|@u2shyz@iPl4pd+XVD!ElrR-bFk|{Ojfn7-R0` zV4cEr;)@hTFvJlnCoJA{?DmW@%hHkH>v8NvEqb+$?NqpLYlSed(4z58{ATrX4ck!<< zZh`=1~VjBzGC-H(!KDIMZa(Lu^x z(kRy;ie^5siB;GU8;Z~s>-2=0H?Wdnnt7!OO{t7V@(GRzdXX!J13s~R&JePIrdZe@ zE=Ug315f;1B{v%WN=+R>MRffx1#vBy?VOKuw;j>8=f*H3A6gf?Z&V?ug z9a@3N%(7t;dV%sZ7X%c0Ru%3q=S6Tndb)vQSw@W*PP=HM^qjd_xl{^7E+x ziA1?dMWU=>Yj4;9&sX`<8UO8UTu5*RLOWM5j`T*Vd#GAN0|<~7WrAKa<%Md|c|_Ti zgG)S`O-u2qlo6gbOASQAS+RGSV}Ya~eW}O)Bb4g_c?v8p864~JQp%(Uwon{My9)Wx z(TXY=u|dr0<$)joI_S=I4#?f*us9^HSLKpk>g50$Po|D)Wl$oIy+2B86(@5(G;tHHx@hkfHTQ=CM#Lk=& zBEWmqgJHm`(o9$$?yzgtJ{3mb!dM*0y@#b%^q(;{_lKD@ZCu!j%ms^-mIA(sjiR(z zVvu8#VBzi$MmAaUe2hq%gAS%h+!8pBs}~AP8Ifu_EwgY>sjTe*0UI;pd3;eDb2I{~ zCfU}ajQPe~A#r!K@+C@aCs_ZD1j?rWm5UPtk4R32=wPB8jX5p}(ovO(PuIpmgs$Qj zC!s`GTOx-<4yP;$wgxB8_BOaijM3}*YXsej5J~TO2{*fRM|9~bp12~1T)u;ICPNCI z2vI+>v#d)5!fGNs_o?B^vQv2Ku_nOR)@-o@mb|w=SJl-jlafrl`0BX)awD6tXtls1 z2%?dM(+IKo4pg&g!@V5@&v3Mn4!?H}ddK@R!PPChlM-iXWaOT<`iF8T-2>FoTSt2K z*#h@kLhF#C%lMEum67<)9N5!VK#>yR45L&@h&&xG(+b7t`{E}7gOAq*5 zXW>OuyG`qCA6h*~CGYr4EXfhm9t9$^Nk>hs;B)#qBIGv$b|viT^ta=nyI#SR zEQV`i0G{|vk-(KZrVbVp9!k$VC-CURRIDBy22t|)JDlRzk+v7z=u1HQ1eGSAl2?=^ zwac#D!$tR(q&x%|=8GZrDOf=FWTExu1$xh%cmT_$q;Yr=(}^W|M3HJsB>(0$_6>TI zYF@`YKssRgqWOv}fAdvGMy=?jwBWBk85kMkuZWYLG`C{2z30T=d3E8On=;u!^JfVr z|MnTdANrgi;vA6wf%yaf!DVdPljni54zndqMgdd*r*{nF6-Lo6Co>kHlpFH{IltgM zNE0YAf-;hoD~Y_YQHV6{I!_P^E!x z=NPEOL>~uYvXp*-wh6v+bELvLxpZbj@n=D27qSpu;q`Y*2yq61dx@ea^fxrk!#OXI zV1iI*ZH0Wpw-H4VAlv7B+xK|OG%7ggeEGq8fY9-)Gq(EbD1MA zQ`C4E#eB`yOQ%PLLve*F;4p4v0~_ciG{`{rXNZzV8&Dx~-*X9aKwT9P7us?+yB1b4 zNHkoMe*Hl(PZ%)&941&G1YLnAiQ>S2N8&LkcvGjhLl6dwBLG*mNHvj&Ah~r@m;)vZ zW$g13zPSHr@#w!2UG_LKo=xbfZ`ma$XvRoA(?P2 zTPPl>bSI$5P-FLjg`@jJZvRlI!F8`x~6>t#u@Ht!3I?e!a8`C11 zFec%}a-4I7M?#F3<1?tBRoRn3{YQ%8p*_3xKd6I?r^1mOf+nv+h8-dU*hmuYrvX3b zQ0YiZ=7l}~9wSq`Raf99lz*{nqJ~l*CM#XhJSFmysuV8z0ETxbMaiOyArKXElaYp? zVb$0!K1oYqBrtAyk$qSKpAmB20YCYNbi^k|GGQgA@pu^MD?TeeT4ktg&)3O&e7{UmSY!748@ zTsJrelXHBG*kZ7emNI!<+R00-2@IxScih*P*CBtcQk(pOB_44g58`tHVNfSGF30f* zNpcAPMWvi5IaB746lEeRz;sWTahf*L51rOf>G2|}sTf}ZnhK;%uo*L-(vd=Gn|%R} zby;ZwieR)Nn-Pef-UvO}D060bb1cImR_G6l(H$gZ2SP?&;q?nd*rNG%R5hAiDnb$m zni=dAXW}>ljJF%8ah*{nfA^s%kL7qxk)cTmo~)M>4go%N_jgRmXN^cpvhyF-r)$6U z4oj6(sc~wu#TiM0EhJ)WqIRQkp%T)#IWnaP!I5$3HIz+*H(v-~0Y@;0@r7}EaDKXw zmJi^LZ>RJsk zfiJenR370dqIeWKWP2>-AP)LB)&>Wm20@Ny7K~?YyjoxolW2M<6y>_3KL9)b*T|Dm zbYT=4DMhjo?D1DOnE?-*D9OW!T>_M$c07-TDyZT`H#1VQV+{-?E+t2*En- zH7Qi&2#7{rEf*m#m?Sbqe#zx{f!evuHh@f*pikAL6|it)!*9(tQsIKGSp#0X5o|7k zSUqYvR;F`;DRzx(E|U2^qFJ3;H=FVv7^D-R9XBKY{SzZ6-6$wyO)GPKs!eemn6c3 zViha-irxZdFq~J8MSvip=n`2hT*QkhU zaDU?i2}xRNJQ$b9P*?O4RaX>X=XeJPyL^$wTT@cOs*r=czC&X}9DBr-ER90kz$Zv_ z%7MRk|0$_RiIxzDV{(KOYS$=WN#$WVVu}`2rj~9SfG~tEozy_b zsU9eWDStAIf5k)r>mIga7r!BWyu}@0%YZn16l*8V{U%65b24X}zyR~cU<|Zk8a6<> z2hW?7N-C3xwmBl?EbIAvb{4e}`f2Vuao=)X4b>C+sYjQfFHge>c_F_H!ooy4?hpj`Y;sz!g<#X3rDN9Tu? zh=Ln1ayD1DiWzecXD0=L)LFt^s&@jON8&9IkN$Kkeq^IYC_-%;iDS}HuvK+waWQZQK z^X?Go*{%6>b1K6k(4w#sx*d3i4djg#{?tJ0cEjU_$Od$R=8vwuf#`u6MY8651b*^b zZxW*^Qob=w;nub&a*&vX#gcMv1mipU?+`(Xk3s1=iBGT)IS}1vM7vw++ktB`M#)S~ zELIL~D|!DVfnRCE&?)P2`k&r$&m*kiO6Wi5=MFcj;8_v? zDIPPc`~ki!2~?DUeDd_F3MY#A+tbSrLWZl3qa|#IDIlgN&P6-k7S=MwD_fsj-w`xT zCd_P^7qJjtAG+Md_javV<=7CmeT-4uHV%DO-@v7htXC>`IEAyhGP&Wiqn-AE{N{jb zya3@mUma@d@l+yp#$)m&gJcrhw)#m0HA(1!JtIMA(hi5R@~&qO>~8s0zw=l<%5(Q$ z6SKjMdgVJ1ciF=GH^m}*2DzP&-APjLL}NcAD9`Fq-n8u~Plg@@S*dAc=iD7bq5f;7 z0F>M%VEN4bxLn=y&T&%@STH#D7wpqE-8n}_cp+=00^|EV|F?MvFhf-CfT#tYy*P>x zL|u*`Sin$#7Bg_v)`T5VL6Rsm<`5}>u;`LNEwC~bdOgSBHX}VMHQnj8Vku-I%7c;V zln5fmt9e|aFzB=z9TB@_ONT7SFEI=#R#Z5Ia}$X*<#H ztrAp>^l0GWiJx{|BSmpbs&4-jw z7=*4QS(1=FSG3^ZoQz4%oP3q?9OJ{u@YFS#qcsWEPAxK%1xfh*jBXN7NSU=f(Yn$r ziZ7NJ23cY!oYM_Elvo%6LaCd$T(o6c6>*bf2>^(kD5WdQNF^pVEJQf`I^rWyTr4J_ zuwiO?DmH1eM){Fdn0DV|O;Se61lH+P$PRU~|DlRwCNLBNT|D`;Uwf??s1H9jU6IUF z5ZwTV5D)-Y);r{x;DsWg>_Q53Rg#(|CViDgKL&MTq9u!k zHN$!H#0HWNkyVwCHB((y)En%jkWmE{twcd)_@#5zEdn`+7&v)>f>(Z5*%W10M7AO1 zg7}04&(+LPaN?6#w;gP;zA`l0#Xko!ALLAiOj|0}SnmW(VVts0Lkx zpn(@&&;e5ck2WG`4w7QLXO&@3;PJR%@CFto#DhVT{Gzu(=)rF>S=Tu_ql&)Q- zL3!)F2HvO#n)M_=-a){|hd~)Z-keQn{{ah#tm3e2TxAuy%BQZz$?(1`&cRf)oJh<>YDO}{Z3yZS(oN@^QhcyTi0IiW z&0+AjDjI=GKoW``HH4&`S}~w8j=_>skP(iqNiSM3f}0_bH<{cp=OGYL$X>c~JnT&G zAl@k2dbp6Bw1wj(E>Op-I3$J40j(iPh@3kdwYLl~FbJIZii|k34*RI$LIWbhVaQ{? z)Zvf=ao~+X9>$l|t&10t|07CtqU1wCD9tMAG0`pjqo~wKYg+X&jMg~V6RcHFA|X<{Uv0Xrl zd@vEh>cp88AdEA|kP)Ds;6I66ax|wn8waOSz5dZjkZTzQCWS{iHrld4mXq8Hdcp+a z*kC0VIR-5JC^Ikp0U)f16{O(uC{N5_S<3^G57rb#Wm3W?`zPO3qc6j3*L42P4u*V?TE>k-xr) zD|@gjnbMdLlgdX(zx)!ZdFlnkc=(i8&NCWLY^RCN;LgWM4I#W^hEdg0)06mbi9H-@ z8$}a`h^$7hIt9=G(}ITDL2{!2fyjmKS+5T;qmM9&kbBbM0dk}?d_am-_>88$Y1tz) z3+Y9a1SS%-Vux%zB9ABG+Ok6e)B<;16B=3xSlC?ZK50SMZn#+6d7a51!Ae2;_<)66 z5RV5&fMD{V|52qZXuzB8a>M}51*AK$B0^204{_<|G%(uCOu@qgPp5|=<`lwoo;*fy zdG`rT5~!5An1f1sdo%_PN?zs_&9-<;-+7(^eO9$q6_^Wkj{xe!J@7@Ot>c{xMYJz_V+;)hS!9;rj{35 zs!S}n2@)y0kss(miiTMR2)-bo9!&R95qRE1e~`X`Vke%cN=!S@^_e2DV~La5#m4H( zJ5nagrSS8JDOAZ1zS>w}9Q+%V_8*6==y0(z!mj^hT2C;l*^Y;nEWQw2hIb1J@Ev*iJJbWfIMY z0Y7k3VTGqgKF2snzvLEMO-3miJq-o^M8fnCazz6RLeHe5*S{9ik1&nw7nh76I|6r` z%j{{&)Y~yg*isT|Tm(vTT9qyt@TB-W7X=cpUg(&A$d>;^C-hl_%e7m5Q9^x?92_(e7R*D z1qF^P%wbIEFFrs{^m%uQXP%@wce^}PdYhe%;HJ;Vj+s&X87sL%mm1H5t>_~D-92S* zjZ|g>$iREpVr_3X7mp=4`b^TFPjmbNmAWrZxdjD}Yf7NB-XlH4fJvy2m_DwK>qjsJ zJ%m?~i(+7725CO>9dB`Gr!DIaE;!@v+1CTf;-5ln&WM4yiebr+}4%eUd;>zEFET<%U_vMn?pKdB|`X$cJ~>Y!_utHV^|a zf;LAZKr(e$We5c&GhZn1Z7hd^27_$2M_($ETYuPJZWT3I$6se8gPfls;xfEPr1t*4K?*o|#OgmZ-mKq!#y0XS}Aa*z;^LuW0vU2UAQvKLSkVsHw1dL2)Bq8+y2g!y;6@m1_L`XT1P+@F5I0#NDg)zhp zu7wF?sFHG6A%7qpMujb9Q=+83?l>9YMAe$TVFn#4hi1d1t0;F$QsG2ro0Wk*~5vl6jI)Awl9` zU1u42fwyVZmVxOJLX*~XnPMu~5Fv;{G(VsoB6Ky1QDd!QWEfYNmWCuC0TQC-K93lK z*wLG}$%YrO;J~oO1A}K&gmMgRqQbK!d zp#M4T;GLXvi{beLj&&~$0cPj9l{3Q{)`1It*h|~Qk`mY-gdw4VWTEAFU$3cp618%v zSq=zzNVeIa=&&@7b4l&VpeX8l8o)|ul`MC0HslZ;Ff$?Y_7VNDNc~nHyU0dF770xV zhf2gQM5&4qsCNc>n{u_H?_f|d`J&?l&nQW~Ys*6y$`0O-iW`lYZO63ekrsip2*O1uV{WE5z9_RRBFj!2g0& zDl2bSLX}yffX99=Aq0L%r6AZO0CTI9iUCrD2p^}FN3$lZC>-~d2kM~(Mc@lw5-&me zX}Z=kI+GjIwmAECCnVRai(;wn#6!<2kmr~K;IfjRplmsS3iihxK_GtM*{Wv~E>6go zYZ-8ZgakXLtr(&>9LJ&f3YIaj7jr}u<=JCq86s0edA-=LurVZY7zmA!3E`J5!KtvE zK?R1TI@C$A-Jlr@D@}HzMESX)T9Op@qk}$|scLtUpCBj~b{+THuPUaWsKP(p@vW4~ z5Oe}0lf*1{u^3VVpcTTA%SMjc8m=byK)S%I4XRw>grhI3n;JWmDs?scQ~wMuqfr2c zIvzQh=7qA}xUxpHH!q8?lS)$9NVC4$u2GwTF;|ZUQ4e20aqBY{frn@gf^OQHZ$hD& z_eilPRS}BBDWr#}YwLP&sv2#R9zevgVEK~+seBn?177=-Ui5s!2xC?P3 zW8)=U1F265*?mSLt0mmeS1$QczrOzZc;`St;Jn1JO8|m38u#zr}v1q z?HfHQ(R;d4xbbU)9G0_zsA&r`D;hVe?4`9K^+{cwP*O(tBMD)h}pk#uS0ZGMrC1fCNepCCh>Ru)}dPeDiT^V%eJc`wu}{BwIQh^AldZ+MV_6GW(+_zXPXj z(ZQoa!^Nv$tkDJ@J5kSLp68aG-MUYyV7_X%b~jhd-#brD_{)*zzFXK#p?gSJ@irCh zI_JWMQ@WUuQ7b}fHiUL99Eh5F5Fgl(ogk-WB5}CMH~%W}Q4|kX1yk`y`QTHtB^HCA zv$CAGMncoW;TQX(l)ASM6-)}%&{Ry_&p{iTlS8Mm0d}Noi=Lzqw-+wz@dh5vOD6@? z8cj1NW^kDm)^W-+)-XxGpcG0Ognh&?@npVo^4Ht!FE55Sw%W)v&|)bK8WK}W=+g+o znr`|raYLPHF#>9{a0hH5lZG`-Zfd8xDv5CiVv_`CTe!9G41^59(!Q0y&FX!a`P;W~ z4W7+pY_ntwqAEvvc5fCWu2EYPJROac5k%34F3J$Tec1BcpkF5jB=$#~f-2KV6!@nd z>Ka*l4HAi**Abto;L3g>kGO;m0>8HDYy1HKeM zJR_AwAP-$3^OMlyyUP{^KX-#FVjN_E;N!zO<1Ovpsm()q#eVRLJUEO?_ zWx~rV)q;n;j68^4_%+pMJn$Ob`!#zsjsMfVMU{;{voImL9nN5-Ex77so)oUS3`{95 zQ0AffIeSXRufEXz(lDsI0kw6_CTr{5t0YFIqfbN}evX2zjt+vM=#7~#zoQ5036qOJ z?FboEzKc5P(4stUe1iMqfss0jbWP>v9K-(Q3-q@$(+4&yJMt`NL&xEKIDxpx}XQLS=+Lb0P=U)j)XBm(m`; zN|QEQ4A&NR^~GJy?Es^ykm2es%KxoXx(u;00)fc_;>*KAg22CQ;HxvE4y(Ar)#4l@ zu*M!~*X^rO83n9_)G-ZMASBiH-aju=^!^p`ti3H%TjdqaD1XD-Wv#l?)K`Oc@rmja zTXuPtHRhk9E^~j5Viz(qTv=l(k67RIW5yqO=k~;Vg-V5IC|q7m&m)b3w)UlU(uyg%V`9 zt9_;nnl2Kz5auU>6xGP)#1N0EWP!6(r=z2=_)Wo|6+8Ng?E153n~Z-I+PgEKK5E0& z^4&D$2QseAk@L=)_ZGQV)&EgJ{WtB305m%xBZ{1K4iza>YSall!520FNK{1jweAbY z@=Sk3aJK6;-Ou`ZFa$YCXjN=M7>m@4*&+g_i(tT0DLE+sY^?wYI9RTg;I`EyjSs;n zI+)(p6l8%n>j-|2*HQ5}`os7u(bRBTGL*y-2Xn)O81(mn2!H|8wNVt(m$mES90M5;jzn35#7>J( zy8Mw4B2NZ}A3OXgp#KDmSRo4qANp_+Eu6@a-tM^p2trsj8&^z)l;DQLs5qupC^JV3 zR2zu-{$S+Pg#=1Nx-O7fXmi8CKqr7aX)>jpp%Zu{ZV>3_P#R3sq)rgD&_hHnSGtir zi}oIq3ooBIFu|y%8Fq21>UgW`=A9JoR?T7-!|2a0PPq)VMQbfelbvoALefC50y&YF zbs76^$)Jh^F|2`hkq#!1Vo#w#i#j!!4s7W-1lhP2u_cL`jA~|7?~<|9?%53N49t~{ zL7%vB@b<#*7=Vr%k#(kDj4;Zh(%B(1PXmarjXM_(@l@ropHyI0jXo0VK?KiBOYxZ4 z6>{0jq*$N%SpS+mxZP`5#|8vKz0-_msobQGE9eXef)s%Cq|XAvsE~p|+J&dkHm-zm ziW|gypbcz4apF~aCVs(|BbW8{5PuZ(H)1Jb_=gh%^;xFCi$hEn7%dZWxKRNm1|&#S z2o7?=Nes%EiCMOQF~wyFF`<_RxJ@(ze?f*Y#v2yIr~*Gqok(Vxt3fl215{|V<#A>m zh0F^(ka9!^D-mcE7C8nvq!L8z(5D(kEh*%ELDo~#9~;^69g$QPHcAIU@VCVrHSFM| znVOPhhB6q$;-N8mn3a@`1WIbejLIEWUXDVFMF)$(^%kfFQq)o$c(grJ-IEkvRwYM2 zs8uUW&HuoFn=N3%ChMloriV^9y_iMap_iyCs2n^9zwV@FbGCL9amk{KuRRvG^`l3)`pLSLE%2|YivADMyJZ0 zHxQCGL@99u>F`Vq9roAJLZ<(kB+7fqg*ChP1udaGU3e+yA(0Tv-r=&FN%+cu*W*A< zVYgvr&B5k)ndR!Lz`4+%9cIQ}N~AiY$mYmpuB<^>NKiqaYH~GPgHPo7?B1hCn7<2+ zkxz1MI61T`)a7Q?A88M_c9wYx4yZ|@`OJ*-`T@Jg8tMq%ARkZ)9zBYQ3{29i?J13N98$2P)H3$-=UkEII(<#dld!nWi6(wH=gTkICn4Fvxd?uKNnL4>}TLl!aS65ZROhmM0L7Zg!P;tNXyQ)06^P>hO% zp;yMj@|78tg;(rp6r-xt05LQ#6Z%66GZI$DNIoKsps-OGOE?+kYf+gL|{2jFqc3TI!tvMbftCJ8A>gAz@o$vqjOY2EP^nI z?jWI*^c3WL0+Ul?*1`;~EYaSqD9?kkMx`umYDXbSf?g!_o6U&XoZPZ338@Jcm(xL& zbnzBNc0w6k8V8}iWI&bxutCmzT}qt_S4cn=n7=$IIC;j&7JYIILba%04mKOH<>NLmiev;E0kgl>@!?thTkp)tfm_G&R|#W_{Nwo-nJL z&%c($w~7B6f)_W#nnw+{xZG9X)J|%~^L;igr`*MEx0k<1d7``QZ7&UkdlqyY?Y!VH z!FkOYrN_E&yYB5Td%wF~X8hAvXHiGK1d&}ubrCAJ^sj>-tRDB(^-{Yz#hpen!MZ5p zk-$|ge<2Lw5c`%&uH5a!FEHmm&<$H;Pd$T5f0MFpK#cma0mN zmwVbVt9i}l3C^Sf74 zVSxuc5H+w;jce}urlAEA$>IXa0_{E$MCCZ1X^5mfGsJ~!T#^lUIJOf^g?G8== F06Q4J6BYmf literal 0 HcmV?d00001 From aefd266f70ad9f5fdec96df488fc5aca99c2fe9d Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Mon, 15 May 2017 10:58:49 +0800 Subject: [PATCH 4/9] Correct text typos and fix NTM's addressing bugs. --- mt_with_external_memory/README.md | 4 +- .../mt_with_external_memory.py | 115 +++++++++--------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index de398b40b7..943b62dfa1 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -99,5 +99,5 @@ TBD ## References 1. Alex Graves, Greg Wayne, Ivo Danihelka, [Neural Turing Machines](https://arxiv.org/abs/1410.5401). arXiv preprint arXiv:1410.5401, 2014. -2. Mingxuan Wang, Zhengdong Lu, Hang Li, Qun Liu，[Memory-enhanced Decoder Neural Machine Translation](https://arxiv.org/abs/1606.02003). arXiv preprint arXiv:1606.02003, 2016. -3. Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio, [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473). arXiv preprint arXiv:1409.0473, 2014. \ No newline at end of file +2. Mingxuan Wang, Zhengdong Lu, Hang Li, Qun Liu, [Memory-enhanced Decoder Neural Machine Translation](https://arxiv.org/abs/1606.02003). arXiv preprint arXiv:1606.02003, 2016. +3. Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio, [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473). arXiv preprint arXiv:1409.0473, 2014. diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py index 5734ed9e13..aee73e2e47 100644 --- a/mt_with_external_memory/mt_with_external_memory.py +++ b/mt_with_external_memory/mt_with_external_memory.py @@ -1,18 +1,5 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. """ - This python script is a example model configuration for neural machine + This python script is an example model configuration for neural machine translation with external memory, based on PaddlePaddle V2 APIs. The "external memory" refers to two types of memories. @@ -21,7 +8,7 @@ Both types of external memories are exploited to enhance the vanilla Seq2Seq neural machine translation. - The implementation largely followers the paper + The implementation primarily follows the paper `Memory-enhanced Decoder for Neural Machine Translation `_, with some minor differences (will be listed in README.md). @@ -39,7 +26,7 @@ hidden_size = 1024 batch_size = 5 memory_slot_num = 8 -beam_size = 40 +beam_size = 3 infer_data_num = 3 @@ -67,24 +54,40 @@ class ExternalMemory(object): :type name: basestring :param mem_slot_size: Size of memory slot/vector. :type mem_slot_size: int - :param boot_layer: Boot layer for initializing memory. Sequence layer - with sequence length indicating the number of memory - slots, and size as mem_slot_size. + :param boot_layer: Boot layer for initializing the external memory. The + sequence layer has sequence length indicating the number + of memory slots, and size as memory slot size. :type boot_layer: LayerOutput :param readonly: If true, the memory is read-only, and write function cannot be called. Default is false. :type readonly: bool + :param enable_interpolation: If set true, the read/write addressing weights + will be interpolated with the weights in the + last step, with the affine coefficients being + a learnable gate function. + :type enable_interpolation: bool """ - def __init__(self, name, mem_slot_size, boot_layer, readonly=False): + def __init__(self, + name, + mem_slot_size, + boot_layer, + readonly=False, + enable_interpolation=True): self.name = name self.mem_slot_size = mem_slot_size self.readonly = readonly + self.enable_interpolation = enable_interpolation self.external_memory = paddle.layer.memory( name=self.name, size=self.mem_slot_size, is_seq=True, boot_layer=boot_layer) + # prepare a constant (zero) intializer for addressing weights + self.zero_addressing_init = paddle.layer.slope_intercept( + input=paddle.layer.fc(input=boot_layer, size=1), + slope=0.0, + intercept=0.0) # set memory to constant when readonly=True if self.readonly: self.updated_external_memory = paddle.layer.mixed( @@ -111,18 +114,18 @@ def __content_addressing__(self, key_vector): size=self.mem_slot_size, act=paddle.activation.Linear(), bias_attr=False) - merged = paddle.layer.addto( + merged_projection = paddle.layer.addto( input=[key_proj_expanded, memory_projection], act=paddle.activation.Tanh()) # softmax addressing weight: w=softmax(v^T a) addressing_weight = paddle.layer.fc( - input=merged, + input=merged_projection, size=1, act=paddle.activation.SequenceSoftmax(), bias_attr=False) return addressing_weight - def __interpolation__(self, key_vector, addressing_weight): + def __interpolation__(self, head_name, key_vector, addressing_weight): """ Interpolate between previous and current addressing weights. """ @@ -134,34 +137,33 @@ def __interpolation__(self, key_vector, addressing_weight): bias_attr=False) # interpolation: w_t = g*w_t+(1-g)*w_{t-1} last_addressing_weight = paddle.layer.memory( - name=self.name + "_addressing_weight", size=1, is_seq=True) - gated_addressing_weight = paddle.layer.addto( - name=self.name + "_addressing_weight", - input=[ - last_addressing_weight, - paddle.layer.scaling(weight=gate, input=addressing_weight), - paddle.layer.mixed( - input=paddle.layer.dotmul_operator( - a=gate, b=last_addressing_weight, scale=-1.0), - size=1) - ], - act=paddle.activation.Tanh()) - return gated_addressing_weight - - def __get_addressing_weight__(self, key_vector): + name=self.name + "_addressing_weight_" + head_name, + size=1, + is_seq=True, + boot_layer=self.zero_addressing_init) + interpolated_weight = paddle.layer.interpolation( + name=self.name + "_addressing_weight_" + head_name, + input=[addressing_weight, addressing_weight], + weight=paddle.layer.expand(input=gate, expand_as=addressing_weight)) + return interpolated_weight + + def __get_addressing_weight__(self, head_name, key_vector): """ Get final addressing weights for read/write heads, including content addressing and interpolation. """ # current content-based addressing addressing_weight = self.__content_addressing__(key_vector) - return addressing_weight # interpolation with previous addresing weight - return self.__interpolation__(key_vector, addressing_weight) + if self.enable_interpolation: + return self.__interpolation__(head_name, key_vector, + addressing_weight) + else: + return addressing_weight def write(self, write_key): """ - Write head for external memory. + Write onto the external memory. It cannot be called if "readonly" set True. :param write_key: Key vector for write heads to generate writing @@ -172,7 +174,7 @@ def write(self, write_key): if self.readonly: raise ValueError("ExternalMemory with readonly=True cannot write.") # get addressing weight for write head - write_weight = self.__get_addressing_weight__(write_key) + write_weight = self.__get_addressing_weight__("write_head", write_key) # prepare add_vector and erase_vector erase_vector = paddle.layer.fc( input=write_key, @@ -205,7 +207,7 @@ def write(self, write_key): def read(self, read_key): """ - Read head for external memory. + Read from the external memory. :param write_key: Key vector for read head to generate addressing signals. @@ -214,7 +216,7 @@ def read(self, read_key): :rtype: LayerOutput """ # get addressing weight for write head - read_weight = self.__get_addressing_weight__(read_key) + read_weight = self.__get_addressing_weight__("read_head", read_key) # read content from external memory scaled = paddle.layer.scaling( weight=read_weight, input=self.updated_external_memory) @@ -227,19 +229,16 @@ def bidirectional_gru_encoder(input, size, word_vec_dim): Bidirectional GRU encoder. """ # token embedding - embeddings = paddle.layer.embedding( - input=input, - size=word_vec_dim, - param_attr=paddle.attr.ParamAttr(name='_encoder_word_embedding')) + embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) # token-level forward and backard encoding for attentions forward = paddle.networks.simple_gru( input=embeddings, size=size, reverse=False) backward = paddle.networks.simple_gru( input=embeddings, size=size, reverse=True) - merged = paddle.layer.concat(input=[forward, backward]) + forward_backward = paddle.layer.concat(input=[forward, backward]) # sequence-level encoding backward_first = paddle.layer.first_seq(input=backward) - return merged, backward_first + return forward_backward, backward_first def memory_enhanced_decoder(input, target, initial_state, source_context, size, @@ -256,9 +255,9 @@ def memory_enhanced_decoder(input, target, initial_state, source_context, size, The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the hidden state vector (or cell state in LSTM) carrying information through a span of sequence time, which is a successful design enriching the model - with capability to "remember" things in the long run. However, such a vector - state is somewhat limited to a very narrow memory bandwidth. External memory - introduced here could easily increase the memory capacity with linear + with the capability to "remember" things in the long run. However, such a + vector state is somewhat limited to a very narrow memory bandwidth. External + memory introduced here could easily increase the memory capacity with linear complexity cost (rather than quadratic for vector state). This enhanced decoder expands its "memory passage" through two @@ -268,7 +267,7 @@ def memory_enhanced_decoder(input, target, initial_state, source_context, size, - Unbounded memory for handling source language's token-wise information. Exactly the attention mechanism over Seq2Seq. - Notice that we take the attention mechanism as a special form of external + Notice that we take the attention mechanism as a particular form of external memory, with read-only memory bank initialized with encoder states, and a read head with content-based addressing (attention). From this view point, we arrive at a better understanding of attention mechanism itself and other @@ -306,12 +305,14 @@ def recurrent_decoder_step(cur_embedding): name="bounded_memory", mem_slot_size=size, boot_layer=bounded_memory_init, - readonly=False) + readonly=False, + enable_interpolation=True) unbounded_memory = ExternalMemory( name="unbounded_memory", mem_slot_size=size * 2, boot_layer=unbounded_memory_init, - readonly=True) + readonly=True, + enable_interpolation=False) # write bounded memory bounded_memory.write(state) # read bounded memory @@ -566,7 +567,7 @@ def infer(): def main(): - paddle.init(use_gpu=False, trainer_count=8) + paddle.init(use_gpu=False, trainer_count=1) train(num_passes=1) infer() From 3ff749b4b49a28c8eee66ba7eca03a4476605eb2 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Mon, 15 May 2017 17:56:17 +0800 Subject: [PATCH 5/9] Complete README.md and add random perturbation to the initial memory in NTM. --- mt_with_external_memory/README.md | 114 +++++++++++++++--- .../mt_with_external_memory.py | 42 +++++-- 2 files changed, 131 insertions(+), 25 deletions(-) diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 943b62dfa1..05b2d17156 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -1,6 +1,6 @@ # Neural Machine Translation with External Memory -带**外部存储或记忆**（External Memory）模块的神经机器翻译模型（Neural Machine Translation, NTM），是神经机器翻译模型的一个重要扩展。它利用可微分（differentiable）的外部记忆模块（其读写控制器以神经网络方式实现），来拓展神经翻译模型内部的工作记忆（Working Memory）的带宽或容量，即作为一个高效的 “外部知识库”，辅助完成翻译等任务中大量信息的临时存储和提取，有效提升模型效果。 +带**外部存储或记忆**（External Memory）模块的神经机器翻译模型（Neural Machine Translation, NTM），是神经机器翻译模型的一个重要扩展。它利用可微分（differentiable）的外部记忆模块（其读写控制器以神经网络方式实现），来拓展神经翻译模型内部的工作记忆（Working Memory）的带宽或容量，即作为一个高效的 “外部知识库”，辅助完成翻译等任务中信息的临时存储和提取，有效提升模型效果。 该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务。例如：机器阅读理解 / 问答（Machine Reading Comprehension / Question Answering)、多轮对话（Multi-turn Dialog）、其他长文本生成任务等。同时，“记忆” 作为认知的重要部分之一，可被用于强化其他多种机器学习模型的表现。该示例仅基于神经机器翻译模型（单指 Seq2Seq， 序列到序列）结合外部记忆机制，起到抛砖引玉的作用，并解释 PaddlePaddle 在搭建此类模型时的灵活性。 @@ -8,6 +8,7 @@ 本文的实现主要参考论文\[[2](#references)\]，但略有不同。并基于 PaddlePaddle V2 APIs。初次使用请参考PaddlePaddle [安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。 + ## Model Overview ### Introduction @@ -20,25 +21,26 @@ 当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个相对稳定的信息通路。带有隐状态向量 $h$（Hidden State 或 Cell State $c$）的 RNN 模型 ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可动态地往上叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 -注意，尽管 $c$ 或者叠加 Gate 结构的 $h$ 的存在，从优化和概率论的角度看通常都有着不同的解释（优化角度：为了在梯度计算中改良 Jacobian 矩阵的特征值，以减轻长程梯度衰减问题，降低优化难度；概率论角度：使得序列具有马尔科夫性），但不妨碍我们从直觉的角度将它理解为某种 “线性” 的且较 “直接” 的 “记忆通道”。 +注意，在 LSTM 中的 $c$ 的引入，或者GRU中 $h$ 的 Leaky 结构的引入，从优化的角度看有着不同的解释（为了在梯度计算中改良 Jacobian 矩阵的特征值，以减轻长程梯度衰减问题，降低优化难度），但不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅。 #### 动态记忆 #2 --- Seq2Seq 中的注意力机制 + 然而这样的一个向量化的 $h$ 的信息带宽极为有限。在 Seq2Seq 序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（encoder）转移至解码器（decoder）的过程中的信息丢失，即通常所说的 “依赖一个状态向量来编码整个源句子，有着严重的信息损失”。 -于是，注意力机制（Attention Mechanism）被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的状态向量，而是依赖一个向量组（其中的每个向量记录编码器处理每个token时的状态向量），并通过软性的（soft）、广泛分布的（distributed) 的注意强度（attention strength) 来分配注意力资源，提取（线性加权）信息用于序列的不同位置的符号生成。 +于是，注意力机制（Attention Mechanism）被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的状态向量，而是依赖一个向量组（其中的每个向量记录编码器处理每个token时的状态向量），并通过软性的、全局分布的的注意强度（attention strengths/ weights) 来分配注意力资源，提取（线性加权）信息用于序列的不同位置的符号生成。这种注意强度的分布，可看成基于内容的寻址，即在源语句的不同位置获取不同的读取强度，起到一种和源语言 “软对齐（Soft Alignment）” 的作用。 -这里的 “向量组” 蕴含着更大更精准的信息，它可以被认为是一个无界的外部存储器（Unbounded External Memory）。在源语言的编码（encoding）完成时，该外部存储即被初始化为各 token 的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于 NTM的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址 (Location-based Addressing)。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，这有待于进一步的探索。另外，这里的 “无界” 指的是 “记忆向量组” 的向量个数非固定，而是随着源语言的 token 数的变化而变化，数量不受限。 +这里的 “向量组” 蕴含着更多更精准的信息，它可以被认为是一个无界的外部存储器（Unbounded External Memory）。在源语言的编码（encoding）完成时，该外部存储即被初始化为各 token 的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于 NTM的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址 (Location-based Addressing)。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，有待进一步的探索。此外，这里的 “无界” 指的是 “记忆向量组” 的向量个数非固定，而是随着源语言的 token 数的变化而变化，数量不受限。 #### 动态记忆 #3 --- 神经图灵机 -图灵机（Turing Machines)，或冯诺依曼体系（Von Neumann Architecture），计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#references)\]试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。 +图灵机（Turing Machines)，或冯诺依曼体系（Von Neumann Architecture），计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#references)\] 试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。

图1. 图灵机（漫画）。
-图灵机的存储机制，被形象比喻成一个纸带（tape），在这个纸带上有读写头（write/read heads）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （contoller) 控制（见图1）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”（$n$为记忆槽/记忆向量的数量，$m$为记忆向量的长度），以前馈神经网络（MLP）或者 循环神经网络（RNN）来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（不赘述，请参考论文\[[1](#references)\]），并最终写入或读出信息，供其他网络使用（见图2）。 +图灵机的存储机制，被形象比喻成一个纸带（tape），在这个纸带上有读写头（write/read heads）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （contoller) 控制（见图1）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”（$n$为记忆槽/记忆向量的数量，$m$为记忆向量的长度），以前馈神经网络（Feedforward Neural Network）或者 循环神经网络（Recurrent Neural Network）来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（不赘述，请参考论文\[[1](#references)\]），并最终写入或读出信息，供其他网络使用（见图2）。

@@ -53,11 +55,12 @@ 当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储（状态-状态转移矩阵）和计算复杂度为代价。而基于神经图灵机的记忆扩展的代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的，而 NTM 的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净。 -所以，在该实现中，RNNs 原有的状态向量 $h$ 或 $c$、 Seq2Seq 常见的注意力机制，被保留；同时，类似 NTM （简化版，无基于位置的寻址） 的有界外部记忆网络被以补充 $h$ 的形式加入。整体的模型实现则类似于论文\[[2](#references)\]，但有少量差异。同时参考该模型的另外一份基于V1 APIs [配置](https://github.com/lcy-seso/paddle_confs_v1/blob/master/mt_with_external_memory/gru_attention_with_external_memory.conf)， 同样有少量差异。具体讨论于 [Further Discussion](#discussions) 一章。 +所以，在该实现中，RNNs 原有的状态向量 $h$ 或 $c$、 Seq2Seq 常见的注意力机制，被保留；同时，类似 NTM （简化版，无基于位置的寻址） 的有界外部记忆网络被以补充 $h$ 的形式加入。整体的模型实现则类似于论文\[[2](#references)\]（网络结构见 [Architecture](#architecture)一章），但有少量差异。同时参考该模型的另外一份基于V1 APIs [配置](https://github.com/lcy-seso/paddle_confs_v1/blob/master/mt_with_external_memory/gru_attention_with_external_memory.conf)， 同样有少量差异。具体讨论于 [Discussions](#discussions) 一章。 注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 ExternalMemory 类。只是前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。 ### Architecture + 网络总体结构基于传统的 Seq2Seq 结构，即RNNsearch\[[3](#references)\] 结构基础上叠加简化版 NTM\[[1](#references)\]。 - 编码器（encoder）采用标准**双向GRU结构**（非 stack），不再赘述。 @@ -72,29 +75,110 @@ 1. $M_{t-1}^B$ 和 $M_t^B$ 为有界外部存储矩阵，前者为上一时间步存储矩阵的状态，后者为当前时间步的状态。$\textrm{read}^B$ 和 $\textrm{write}$ 为对应的读写头（包含其控制器）。$r_t$ 为对应的读出向量。 2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头（无 写头），二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量（即 attention context)。 -3. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#references)\] 基本一致。 +3. $y_{t-1}$ 为解码器上一步的输出字符并做词嵌入（Word Embedding)，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 +4. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#references)\] 基本一致（略有不一致之处为：用于 attention 的 decoder state 被改进，即叠加了一 Tanh 隐层并引入了 $y_{t-1}$）。 + ## Implementation -TBD +整体实现的关键部分在辅助模型类`ExternalMemory` 和模型配置函数 `memory_enhanced_seq2seq`。 + +### `ExternalMemory` Class + +`ExternalMemory` 类实现通用的简化版**神经图灵机**。相比完整版神经图灵机，该类仅实现了基于内容的寻址（Content Addressing, Interpolation），不包括基于位置的寻址（ Convolutional Shift, Sharpening)。读者可以自行将其补充成为一个完整的神经图灵机。 + +类结构如下： + +``` +class ExternalMemory(object): + __init__(self, name, mem_slot_size, boot_layer, readonly, enable_projection) + __content_addressing__(self, key_vector) + __interpolation__(self, head_name, key_vector, addressing_weight) + __get_adressing_weight__(self, head_name, key_vector) + write(self, write_key) + read(self, read_key) +``` + +神经图灵机的 “外部存储” 采用 `Paddle.layer.memory`实现，注意这里的`is_seq`需要设成`True`, 该序列的长度表示记忆槽的数量（$n$), `size` 表示记忆槽（向量）的大小。同时依赖一个外部层作为初始化， $n$ 取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 + +`ExternalMemory`类的寻址逻辑通过 `__content_addressing__` 和 `__interpolation__` 两个私有函数实现。读和写操作通过 `read` 和 `write` 两个函数实现。并且读和写的寻址独立进行，不同于 \[[2](#references)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 + +为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中，并且采用简单的前馈网络。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 + +`ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。便于用该类等价实现传统的注意力机制。 + +注意， `ExternalMemory` 仅可用于和 `paddle.layer.recurrent_group`配合使用（使用在其用户自定义的 `step` 函数中）。它不可以单独存在。 + +### `memory_enhanced_seq2seq` Related Functions + +涉及三个主要函数： + +``` +memory_enhanced_seq2seq(...) +bidirectional_gru_encoder(...) +memory_enhanced_decoder(...) +``` + +`memory_enhanced_seq2seq` 函数定义整个带外部存储增强的序列到序列模型，是模型定义的主调函数。`memory_enhanced_seq2seq` 首先调用`bidirectional_gru_encoder` 对源语言进行编码，然后通过 `memory_enhanced_decoder` 进行解码。 + +`bidirectional_gru_encoder` 函数实现双向单层 GRU（Gated Recurrent Unit） 编码器。返回两组（均为`LayerOutput`）结果：一组为向量序列（包含所有字符的前后向编码结果），一组为整个源语句的最终编码向量（仅包含后向结果）。前者用于解码器的注意力机制，后者用于初始化解码器的状态向量。 + +`memory_enhanced_decoder` 函数实现通过外部记忆增强的 GRU 解码器。它利用同一个`ExternalMemory` 类实现两种外部记忆模块： + +- 无界外部记忆：即传统的注意力机制（Attention）。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 的`boot_layer`（用作存储矩阵的初始化）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符（token）数。 +- 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值 Pooling 后并扩展为指定序列长度后，作为 `ExternalMemory` 的`boot_layer`。因此，该存储的记忆槽数目是固定的，并且其所有记忆槽的初始化内容均相同。 + +注意，注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 + +此外，在该实现中，将 `ExternalMemory` 的 `write` 操作提前至 `read` 之前，以避开潜在的拓扑连接局限，详见 [Issue](https://github.com/PaddlePaddle/Paddle/issues/2061)。我们可以看到，本质上他们是等价的。 ## Getting Started ### Prepare the Training Data -TBD +数据是通过无参的 `reader()` 函数，进入训练过程。因此我们需要为训练数据和测试数据分别构造 `reader()` 函数。`reader()` 函数使用 `yield` 来保证可迭代性（iterable，即可以 `for instance in reader()` 方式迭代运行）, 例如 + +``` +def data_reader(): + for instance in data_list: + yield instance +``` + +`yield` 返回的每条样本需为三元组，分别包含编码器输入字符ID列表（即源语言序列），解码器输入字符ID列表（即目标语言序列，且序列右移一位），解码器输出字符ID列表（即目标语言序列）。 + +用户需自行完成字符的切分 (tokenize) ，并构建字典完成 ID 化。 -### Training a Model +PaddlePaddle 的接口 paddle.paddle.wmt14 ([wmt14.py](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py))， 默认提供了一个经过预处理的[较小规模的 wmt14 子集](http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz)。并提供了两个reader creator函数如下： -TBD +``` +paddle.dataset.wmt14.train(dict_size) +paddle.dataset.wmt14.test(dict_size) +``` -### Generating Sequences +这两个函数被调用时即返回相应的`reader()`函数，供`paddle.traner.SGD.train`使用。 -TBD +当我们需要使用其他数据时，可参考 paddle.paddle.wmt14 ([wmt14.py](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)) 构造相应的 data creator，并替换 `paddle.dataset.wmt14.train` 和 `paddle.dataset.wmt14.train` 成相应函数名。 + +### Running the script + +命令行输入： + +```Python mt_with_external_memory.py``` + +即可运行训练脚本（默认训练一轮），训练模型将被定期保存于本地 `params.tar.gz`。训练完成后，将为少量样本生成翻译结果，详见 `infer` 函数。 + +## Experiments + +To Be Added. ## Discussions -TBD +#### 和论文\[[2](#references)\]实现的差异 + +差异如下： + +1. 基于内容的寻址公式不同，原文为 $a = v^T(WM^B + Us)$, 本实现为 $a = v^T \textrm{tanh}(WM^B + Us)$，以保持和 \[[3](#references)\] 中的注意力机制寻址方式一致。 +2. 有界外部存储的初始化方式不同，原文为：$M^B = \sigma(W\sum_{i=0}^{i=n}h_i)/n + V$, $V_{i,j}~\in \mathcal{N}(0, 0.1)$。本实现暂为 $M^B = \sigma(\frac{1}{n}W\sum_{i=0}^{i=n}h_i) + V$。 ## References diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py index aee73e2e47..22f96a72ce 100644 --- a/mt_with_external_memory/mt_with_external_memory.py +++ b/mt_with_external_memory/mt_with_external_memory.py @@ -20,6 +20,7 @@ import paddle.v2 as paddle import sys import gzip +import random dict_size = 30000 word_vec_dim = 512 @@ -28,6 +29,7 @@ memory_slot_num = 8 beam_size = 3 infer_data_num = 3 +memory_perturb_stddev = 0.1 class ExternalMemory(object): @@ -46,6 +48,10 @@ class ExternalMemory(object): handling. Besides, some existing mechanism can be realized directly with the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an unbounded external memory). + + Besides, the ExternalMemory class must be used together with + paddle.layer.recurrent_group (within its step function). It can never be + used in a standalone manner. For more details, please refer to `Neural Turing Machines `_. @@ -289,11 +295,17 @@ def memory_enhanced_decoder(input, target, initial_state, source_context, size, input=source_context, pooling_type=paddle.pooling.Avg()), size=size, act=paddle.activation.Sigmoid()) - bounded_memory_init = paddle.layer.expand( - input=bounded_memory_slot_init, - expand_as=paddle.layer.data( - name='bounded_memory_template', - type=paddle.data_type.integer_value_sequence(0))) + bounded_memory_perturbation = paddle.layer.data( + name='bounded_memory_perturbation', + type=paddle.data_type.dense_vector_sequence(size)) + bounded_memory_init = paddle.layer.addto( + input=[ + paddle.layer.expand( + input=bounded_memory_slot_init, + expand_as=bounded_memory_perturbation), + bounded_memory_perturbation + ], + act=paddle.activation.Linear()) unbounded_memory_init = source_context # prepare step function for reccurent group @@ -477,17 +489,22 @@ def train(num_passes): "source_words": 0, "target_words": 1, "target_next_words": 2, - "bounded_memory_template": 3 + "bounded_memory_perturbation": 3 } + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [ + [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] + for j in xrange(memory_slot_num) + ] train_append_reader = reader_append_wrapper( reader=paddle.dataset.wmt14.train(dict_size), - append_tuple=([0] * memory_slot_num, )) + append_tuple=(bounded_memory_perturbation, )) train_batch_reader = paddle.batch( reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), batch_size=batch_size) test_append_reader = reader_append_wrapper( reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=([0] * memory_slot_num, )) + append_tuple=(bounded_memory_perturbation, )) test_batch_reader = paddle.batch( reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), batch_size=batch_size) @@ -524,7 +541,7 @@ def infer(): source_words = paddle.layer.data( name="source_words", type=paddle.data_type.integer_value_sequence(dict_size)) - beam_gen = seq2seq( + beam_gen = memory_enhanced_seq2seq( encoder_input=source_words, decoder_input=None, decoder_target=None, @@ -540,9 +557,14 @@ def infer(): # prepare infer data infer_data = [] + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [ + [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] + for j in xrange(memory_slot_num) + ] test_append_reader = reader_append_wrapper( reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=([0] * memory_slot_num, )) + append_tuple=(bounded_memory_perturbation, )) for i, item in enumerate(test_append_reader()): if i < infer_data_num: infer_data.append((item[0], item[3], )) From 56dd1e41ab91318272102a2745a441bcc36bd6bc Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Sun, 21 May 2017 20:04:19 +0800 Subject: [PATCH 6/9] Update README.md to improve readability. --- mt_with_external_memory/README.md | 152 +++++++++++++++--------------- 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 05b2d17156..583ab570bb 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -1,89 +1,93 @@ -# Neural Machine Translation with External Memory +# 带外部记忆机制的神经机器翻译 -带**外部存储或记忆**（External Memory）模块的神经机器翻译模型（Neural Machine Translation, NTM），是神经机器翻译模型的一个重要扩展。它利用可微分（differentiable）的外部记忆模块（其读写控制器以神经网络方式实现），来拓展神经翻译模型内部的工作记忆（Working Memory）的带宽或容量，即作为一个高效的 “外部知识库”，辅助完成翻译等任务中信息的临时存储和提取，有效提升模型效果。 +带**外部记忆**（External Memory）机制的神经机器翻译模型（Neural Machine Translation, NMT），是神经机器翻译模型的一个重要扩展。它利用可微分的外部记忆网络，来拓展神经翻译模型内部工作记忆（Working Memory）的容量或带宽，即引入一个高效的 “外部知识库”，辅助完成翻译等任务中信息的临时存取，有效改善模型表现。 -该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务。例如：机器阅读理解 / 问答（Machine Reading Comprehension / Question Answering)、多轮对话（Multi-turn Dialog）、其他长文本生成任务等。同时，“记忆” 作为认知的重要部分之一，可被用于强化其他多种机器学习模型的表现。该示例仅基于神经机器翻译模型（单指 Seq2Seq， 序列到序列）结合外部记忆机制，起到抛砖引玉的作用，并解释 PaddlePaddle 在搭建此类模型时的灵活性。 +该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务，例如：机器阅读理解 / 问答、多轮对话、其他长文本生成等。同时，“记忆” 作为认知的重要部分之一，可用于强化其他多种机器学习模型的表现。 -本文所采用的外部记忆机制，主要指**神经图灵机**\[[1](#references)\]。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度神经网络的背景下，涌现出一系列有意思的工作。例如：记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。除神经图灵机外，其他均不在本文的讨论范围内。 +本文所采用的外部记忆机制，主要指**神经图灵机** \[[1](#参考文献)\]，将于后文详细描述。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度学习的背景下，涌现出一系列有价值的工作，例如：记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。除神经图灵机外，其他均不在本文的讨论范围内。 -本文的实现主要参考论文\[[2](#references)\]，但略有不同。并基于 PaddlePaddle V2 APIs。初次使用请参考PaddlePaddle [安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。 +本文的实现主要参考论文\[[2](#参考文献)\]。本文假设读者已充分阅读并理解PaddlePaddle Book中[机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章。 -## Model Overview +## 模型概述 -### Introduction +### 记忆机制简介 -记忆（Memory)，是人类（或动物）认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（不同于感知）成为可能。记忆，同样是机器学习模型需要拥有的关键能力之一。 +记忆（Memory)，是人类认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（不同于感知）成为可能。记忆，同样是机器学习模型需要拥有的关键能力之一。 -可以说，任何机器学习模型，原生就拥有一定的记忆能力，无论它是参数模型，还是非参模型，无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络参数即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的，在预测时，模型是静态一致的，不拥有额外的跨时间步的记忆能力。 +可以说，任何机器学习模型，原生就拥有一定的记忆能力：无论它是参数模型（模型参数即记忆），还是非参模型（样本即记忆）；无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络连接权值即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的；在预测时，模型是静态一致的，不拥有额外的跨时间步的信息记忆能力。 -#### 动态记忆 #1 --- RNNs 中的隐状态向量 +#### 动态记忆 1 --- RNNs 中的隐状态向量 -当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个相对稳定的信息通路。带有隐状态向量 $h$（Hidden State 或 Cell State $c$）的 RNN 模型 ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可动态地往上叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 +当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个可持久的信息通路。带有隐状态向量 $h$（或 LSTM 中的细胞状态向量 $c$）的循环神经网络（Recurrent Neural Networks, RNNs） ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 -注意，在 LSTM 中的 $c$ 的引入，或者GRU中 $h$ 的 Leaky 结构的引入，从优化的角度看有着不同的解释（为了在梯度计算中改良 Jacobian 矩阵的特征值，以减轻长程梯度衰减问题，降低优化难度），但不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅。 +我们注意到，LSTM 中的细胞状态向量 $c$ 的引入，或者 GRU 中状态向量 $h$ 的以门（Gate）控制的线性跨层结构（Leaky Unit）的引入，从优化的角度看有着不同的理解：即为了梯度计算中各时间步的一阶偏导矩阵（雅克比矩阵）的谱分布更接近单位阵，以减轻长程梯度衰减问题，降低优化难度。但这不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅，如图1（引自[此文](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)）所示的 LSTM 中的细胞状态向量 $c$ 可视为这样一个用于信息持久化的 “线性记忆通道”。 -#### 动态记忆 #2 --- Seq2Seq 中的注意力机制 +
+
+图1. LSTM 中的细胞状态向量作为 “记忆通道” 示意图 +
+ +#### 动态记忆 2 --- Seq2Seq 中的注意力机制 -然而这样的一个向量化的 $h$ 的信息带宽极为有限。在 Seq2Seq 序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（encoder）转移至解码器（decoder）的过程中的信息丢失，即通常所说的 “依赖一个状态向量来编码整个源句子，有着严重的信息损失”。 +然而这样的一个向量化的 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着一定程度的信息丢失。 -于是，注意力机制（Attention Mechanism）被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的状态向量，而是依赖一个向量组（其中的每个向量记录编码器处理每个token时的状态向量），并通过软性的、全局分布的的注意强度（attention strengths/ weights) 来分配注意力资源，提取（线性加权）信息用于序列的不同位置的符号生成。这种注意强度的分布，可看成基于内容的寻址，即在源语句的不同位置获取不同的读取强度，起到一种和源语言 “软对齐（Soft Alignment）” 的作用。 +于是，注意力机制（Attention Mechanism）\[[3](#参考文献)\] 被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量，而是依赖一个向量组，向量组中的每个向量为编码器的各字符（Tokens）级编码向量（状态向量），并通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式提权信息用于序列的不同位置的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容获取不同的读取强度，起到一种和源语言 “软对齐（Soft Alignment）” 的作用。 -这里的 “向量组” 蕴含着更多更精准的信息，它可以被认为是一个无界的外部存储器（Unbounded External Memory）。在源语言的编码（encoding）完成时，该外部存储即被初始化为各 token 的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于 NTM的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址 (Location-based Addressing)。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，有待进一步的探索。此外，这里的 “无界” 指的是 “记忆向量组” 的向量个数非固定，而是随着源语言的 token 数的变化而变化，数量不受限。 +这里的 “向量组” 蕴含着更多更精准的信息，它可以被认为是一个无界的外部记忆模块（Unbounded External Memory）。“无界” 指的是向量组的向量个数非固定，而是随着源语言的字符数的变化而变化，数量不受限。在源语言的编码完成时，该外部存储即被初始化为各字符的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于神经图灵机的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址（Location-based Addressing）。两种寻址方式不赘述，详见 \[[1](#参考文献)\]。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，有待进一步的探索。 -#### 动态记忆 #3 --- 神经图灵机 +#### 动态记忆 3 --- 神经图灵机 -图灵机（Turing Machines)，或冯诺依曼体系（Von Neumann Architecture），计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#references)\] 试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。 +图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。

-图1. 图灵机（漫画）。 +图2. 图灵机结构漫画
-图灵机的存储机制，被形象比喻成一个纸带（tape），在这个纸带上有读写头（write/read heads）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （contoller) 控制（见图1）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”（$n$为记忆槽/记忆向量的数量，$m$为记忆向量的长度），以前馈神经网络（Feedforward Neural Network）或者 循环神经网络（Recurrent Neural Network）来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（不赘述，请参考论文\[[1](#references)\]），并最终写入或读出信息，供其他网络使用（见图2）。 +图灵机的存储机制，常被形象比喻成一个纸带（Tape），在这个纸带上有读头（Read Head）和 写头（Write Head）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （Contoller) 控制（见图2，引自[此处](http://www.worldofcomputing.net/theory/turing-machine.html)）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”，其中 $n$ 为记忆向量（又成记忆槽）的数量，$m$为记忆向量的长度，以前馈神经网络或循环神经网络来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（寻址方式不赘述，请参考论文\[[1](#参考文献)\]），并最终写入或读出信息，供其他网络使用。神经图灵机结构示意图，见图3，引自\[[1](#参考文献)\]。

-图2. 神经图灵机结构示意图。 +图3. 神经图灵机结构示意图
-和上述的注意力机制相比，神经图灵机有着很多相同点和不同点。相同在于：均利用外部存储和其上的相关操作，矩阵（或向量组）形式的存储，可微分的寻址方式。不同在于：神经图灵机有读有写（真正意义上的存储器），并且其寻址不仅限于基于内容的寻址，同时结合基于位置的寻址（使得例如 “长序列复制” 等需要 “连续寻址” 的任务更容易），此外它是有界的（bounded)。 +和上述的注意力机制相比，神经图灵机有着诸多相同点和不同点。相同在于：均利用矩阵（或向量组）形式的存储，可微分的寻址方式。不同在于：神经图灵机有读有写（是真正意义上的存储器），并且其寻址不仅限于基于内容的寻址，同时结合基于位置的寻址（使得例如 “长序列复制” 等需要 “连续寻址” 的任务更容易），此外它是有界的（Bounded)；而注意机制仅仅有读操作，无写操作，并且仅基于内容寻址，此外它是无界的（Unbounded）。 #### 三种记忆混合，强化神经机器翻译模型 -尽管在一般的 Seq2Seq 模型中，注意力机制都已经是标配。然而，注意机制的外部存储仅仅是用于存储源语言的信息。在解码器内部，信息通路仍然是依赖于 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充（或替换）解码器内部的单向量信息通路，成为自然而然的想法。 +尽管在一般的序列到序列模型中，注意力机制已经是标配。然而，注意机制的外部存储仅仅是用于存储源语言的字符级信息。在解码器内部，信息通路仍然是依赖于 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充解码器内部的单向量信息通路，成为自然而然的想法。 -当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储（状态-状态转移矩阵）和计算复杂度为代价。而基于神经图灵机的记忆扩展的代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的，而 NTM 的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净。 +当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储和计算复杂度为代价（状态-状态转移矩阵）。而基于神经图灵机的记忆扩展代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。另外值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的；而神经图灵机的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净，干扰更小。 -所以，在该实现中，RNNs 原有的状态向量 $h$ 或 $c$、 Seq2Seq 常见的注意力机制，被保留；同时，类似 NTM （简化版，无基于位置的寻址） 的有界外部记忆网络被以补充 $h$ 的形式加入。整体的模型实现则类似于论文\[[2](#references)\]（网络结构见 [Architecture](#architecture)一章），但有少量差异。同时参考该模型的另外一份基于V1 APIs [配置](https://github.com/lcy-seso/paddle_confs_v1/blob/master/mt_with_external_memory/gru_attention_with_external_memory.conf)， 同样有少量差异。具体讨论于 [Discussions](#discussions) 一章。 +所以，在该示例的实现中，RNN 原有的状态向量和注意力机制被保留；同时，基于简化版的神经图灵机的有界外部记忆机制被引入以补充解码器单状态向量记忆。整体的模型实现参考论文\[[2](#参考文献)\]，但有少量差异，详见[其他讨论](#其他讨论)一章。 -注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 ExternalMemory 类。只是前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。 -### Architecture +### 模型网络结构 -网络总体结构基于传统的 Seq2Seq 结构，即RNNsearch\[[3](#references)\] 结构基础上叠加简化版 NTM\[[1](#references)\]。 +网络总体结构在带注意机制的序列到序列结构（即RNNsearch\[[3](##参考文献)\]） 基础上叠加简化版神经图灵机\[[1](#参考文献)\]外部记忆模块。 -- 编码器（encoder）采用标准**双向GRU结构**（非 stack），不再赘述。 -- 解码器（decoder）采用和论文\[[2](#references)\] 基本相同的结构，见图3（修改自论文\[[2](#references)\]) 。 +- 编码器（Encoder）采用标准**双向 GRU 结构**（非 stack），不赘述。 +- 解码器（Decoder）采用和论文\[[2](#参考文献)\] 基本相同的结构，见图4（修改自论文\[[2](#参考文献参考文献)\]) 。

-图3. 通过外部记忆增强的解码器结构示意图。 +图4. 通过外部记忆增强的解码器结构示意图
解码器结构图，解释如下： 1. $M_{t-1}^B$ 和 $M_t^B$ 为有界外部存储矩阵，前者为上一时间步存储矩阵的状态，后者为当前时间步的状态。$\textrm{read}^B$ 和 $\textrm{write}$ 为对应的读写头（包含其控制器）。$r_t$ 为对应的读出向量。 -2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头（无 写头），二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量（即 attention context)。 -3. $y_{t-1}$ 为解码器上一步的输出字符并做词嵌入（Word Embedding)，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 -4. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#references)\] 基本一致（略有不一致之处为：用于 attention 的 decoder state 被改进，即叠加了一 Tanh 隐层并引入了 $y_{t-1}$）。 +2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头，二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量。 +3. $y_{t-1}$ 为解码器上一步的输出字符并做词向量（Word Embedding)，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 +4. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#参考文献)\] 基本一致（略有不一致之处为：用于 attention 的 decoder state 被改进，即叠加了一隐层并引入了 $y_{t-1}$）。 -## Implementation +## 算法实现 -整体实现的关键部分在辅助模型类`ExternalMemory` 和模型配置函数 `memory_enhanced_seq2seq`。 +算法实现的关键部分在辅助类`ExternalMemory` 和模型函数 `memory_enhanced_seq2seq`。 -### `ExternalMemory` Class +### `ExternalMemory` 类 `ExternalMemory` 类实现通用的简化版**神经图灵机**。相比完整版神经图灵机，该类仅实现了基于内容的寻址（Content Addressing, Interpolation），不包括基于位置的寻址（ Convolutional Shift, Sharpening)。读者可以自行将其补充成为一个完整的神经图灵机。 @@ -91,25 +95,25 @@ ``` class ExternalMemory(object): - __init__(self, name, mem_slot_size, boot_layer, readonly, enable_projection) - __content_addressing__(self, key_vector) - __interpolation__(self, head_name, key_vector, addressing_weight) - __get_adressing_weight__(self, head_name, key_vector) - write(self, write_key) - read(self, read_key) + __init__(self, name, mem_slot_size, boot_layer, readonly, enable_projection) + __content_addressing__(self, key_vector) + __interpolation__(self, head_name, key_vector, addressing_weight) + __get_adressing_weight__(self, head_name, key_vector) + write(self, write_key) + read(self, read_key) ``` -神经图灵机的 “外部存储” 采用 `Paddle.layer.memory`实现，注意这里的`is_seq`需要设成`True`, 该序列的长度表示记忆槽的数量（$n$), `size` 表示记忆槽（向量）的大小。同时依赖一个外部层作为初始化， $n$ 取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 +神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，注意这里的`is_seq`需设成`True`，该序列的长度表示记忆槽的数量，`size` 表示记忆槽（向量）的大小。同时依赖一个外部层作为初始化， 记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 -`ExternalMemory`类的寻址逻辑通过 `__content_addressing__` 和 `__interpolation__` 两个私有函数实现。读和写操作通过 `read` 和 `write` 两个函数实现。并且读和写的寻址独立进行，不同于 \[[2](#references)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 +`ExternalMemory`类的寻址逻辑通过 `__content_addressing__` 和 `__interpolation__` 两个私有函数实现。读和写操作通过 `read` 和 `write` 两个函数实现。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 -为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中，并且采用简单的前馈网络。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 +为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中，并且采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 `ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。便于用该类等价实现传统的注意力机制。 -注意， `ExternalMemory` 仅可用于和 `paddle.layer.recurrent_group`配合使用（使用在其用户自定义的 `step` 函数中）。它不可以单独存在。 +注意， `ExternalMemory` 只能和 `paddle.layer.recurrent_group`配合使用，具体在用户自定义的 `step` 函数中使用，它不可以单独存在。 -### `memory_enhanced_seq2seq` Related Functions +### `memory_enhanced_seq2seq` 及相关函数 涉及三个主要函数： @@ -119,36 +123,36 @@ bidirectional_gru_encoder(...) memory_enhanced_decoder(...) ``` -`memory_enhanced_seq2seq` 函数定义整个带外部存储增强的序列到序列模型，是模型定义的主调函数。`memory_enhanced_seq2seq` 首先调用`bidirectional_gru_encoder` 对源语言进行编码，然后通过 `memory_enhanced_decoder` 进行解码。 +`memory_enhanced_seq2seq` 函数定义整个带外部记忆机制的序列到序列模型，是模型定义的主调函数。它首先调用`bidirectional_gru_encoder` 对源语言进行编码，然后通过 `memory_enhanced_decoder` 进行解码。 -`bidirectional_gru_encoder` 函数实现双向单层 GRU（Gated Recurrent Unit） 编码器。返回两组（均为`LayerOutput`）结果：一组为向量序列（包含所有字符的前后向编码结果），一组为整个源语句的最终编码向量（仅包含后向结果）。前者用于解码器的注意力机制，后者用于初始化解码器的状态向量。 +`bidirectional_gru_encoder` 函数实现双向单层 GRU（Gated Recurrent Unit） 编码器。返回两组结果：一组为字符级编码向量序列（包含前后向），一组为整个源语句的句级编码向量（仅后向）。前者用于解码器的注意力机制中记忆矩阵的初始化，后者用于解码器的状态向量的初始化。 `memory_enhanced_decoder` 函数实现通过外部记忆增强的 GRU 解码器。它利用同一个`ExternalMemory` 类实现两种外部记忆模块： -- 无界外部记忆：即传统的注意力机制（Attention）。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 的`boot_layer`（用作存储矩阵的初始化）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符（token）数。 -- 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值 Pooling 后并扩展为指定序列长度后，作为 `ExternalMemory` 的`boot_layer`。因此，该存储的记忆槽数目是固定的，并且其所有记忆槽的初始化内容均相同。 +- 无界外部记忆：即传统的注意力机制。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符数。 +- 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值池化（pooling）后并扩展为指定序列长度后，叠加随机噪声（训练和推断时保持一致），作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是固定的。 -注意，注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 +注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 `ExternalMemory` 类。前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “注意机制” 和 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 此外，在该实现中，将 `ExternalMemory` 的 `write` 操作提前至 `read` 之前，以避开潜在的拓扑连接局限，详见 [Issue](https://github.com/PaddlePaddle/Paddle/issues/2061)。我们可以看到，本质上他们是等价的。 -## Getting Started +## 快速开始 -### Prepare the Training Data +### 数据自定义 -数据是通过无参的 `reader()` 函数，进入训练过程。因此我们需要为训练数据和测试数据分别构造 `reader()` 函数。`reader()` 函数使用 `yield` 来保证可迭代性（iterable，即可以 `for instance in reader()` 方式迭代运行）, 例如 +数据是通过无参的 `reader()` 迭代器函数，进入训练过程。因此我们需要为训练数据和测试数据分别构造两个 `reader()` 迭代器。`reader()` 函数使用 `yield` 来实现迭代器功能（即可通过 `for instance in reader()` 方式迭代运行）， 例如 ``` -def data_reader(): - for instance in data_list: - yield instance +def reader(): + for instance in data_list: + yield instance ``` -`yield` 返回的每条样本需为三元组，分别包含编码器输入字符ID列表（即源语言序列），解码器输入字符ID列表（即目标语言序列，且序列右移一位），解码器输出字符ID列表（即目标语言序列）。 +`yield` 返回的每条样本需为三元组，分别包含编码器输入字符列表（即源语言序列，需 ID 化），解码器输入字符列表（即目标语言序列，需 ID 化，且序列右移一位），解码器输出字符列表（即目标语言序列，需 ID 化）。 -用户需自行完成字符的切分 (tokenize) ，并构建字典完成 ID 化。 +用户需自行完成字符的切分 (Tokenize) ，并构建字典完成 ID 化。 -PaddlePaddle 的接口 paddle.paddle.wmt14 ([wmt14.py](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py))， 默认提供了一个经过预处理的[较小规模的 wmt14 子集](http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz)。并提供了两个reader creator函数如下： +PaddlePaddle 的接口 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)， 默认提供了一个经过预处理的、较小规模的 wmt14 英法翻译数据集的子集。并提供了两个reader creator函数如下： ``` paddle.dataset.wmt14.train(dict_size) @@ -157,31 +161,29 @@ paddle.dataset.wmt14.test(dict_size) 这两个函数被调用时即返回相应的`reader()`函数，供`paddle.traner.SGD.train`使用。 -当我们需要使用其他数据时，可参考 paddle.paddle.wmt14 ([wmt14.py](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)) 构造相应的 data creator，并替换 `paddle.dataset.wmt14.train` 和 `paddle.dataset.wmt14.train` 成相应函数名。 +当我们需要使用其他数据时，可参考 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py) 构造相应的 data creator，并替换 `paddle.dataset.wmt14.train` 和 `paddle.dataset.wmt14.train` 成相应函数名。 -### Running the script +### 训练及预测 命令行输入： -```Python mt_with_external_memory.py``` +```python mt_with_external_memory.py``` 即可运行训练脚本（默认训练一轮），训练模型将被定期保存于本地 `params.tar.gz`。训练完成后，将为少量样本生成翻译结果，详见 `infer` 函数。 -## Experiments - -To Be Added. - -## Discussions +## 其他讨论 -#### 和论文\[[2](#references)\]实现的差异 +#### 和论文\[[2](#参考文献)\]实现的差异 差异如下： -1. 基于内容的寻址公式不同，原文为 $a = v^T(WM^B + Us)$, 本实现为 $a = v^T \textrm{tanh}(WM^B + Us)$，以保持和 \[[3](#references)\] 中的注意力机制寻址方式一致。 -2. 有界外部存储的初始化方式不同，原文为：$M^B = \sigma(W\sum_{i=0}^{i=n}h_i)/n + V$, $V_{i,j}~\in \mathcal{N}(0, 0.1)$。本实现暂为 $M^B = \sigma(\frac{1}{n}W\sum_{i=0}^{i=n}h_i) + V$。 +1. 基于内容的寻址公式不同: 原文为 $a = v^T(WM^B + Us)$，本示例为 $a = v^T \textrm{tanh}(WM^B + Us)$，以保持和 \[[3](#参考文献)\] 中的注意力机制寻址方式一致。 +2. 有界外部存储的初始化方式不同: 原文为 $M^B = \sigma(W\sum_{i=0}^{i=n}h_i)/n + V$, $V_{i,j}~\in \mathcal{N}(0, 0.1)$，本示例为 $M^B = \sigma(\frac{1}{n}W\sum_{i=0}^{i=n}h_i) + V$。 +3. 外部记忆机制的读和写的寻址逻辑不同：原文二者共享同一个寻址强度，相当于权值联结（Weight Tying）正则。本示例不施加该正则，读和写采用独立寻址。 +4. 同时间步内的外部记忆读写次序不同：原文为先读后写，本示例为先写后读，本质等价。 -## References +## 参考文献 1. Alex Graves, Greg Wayne, Ivo Danihelka, [Neural Turing Machines](https://arxiv.org/abs/1410.5401). arXiv preprint arXiv:1410.5401, 2014. -2. Mingxuan Wang, Zhengdong Lu, Hang Li, Qun Liu, [Memory-enhanced Decoder Neural Machine Translation](https://arxiv.org/abs/1606.02003). arXiv preprint arXiv:1606.02003, 2016. +2. Mingxuan Wang, Zhengdong Lu, Hang Li, Qun Liu, [Memory-enhanced Decoder Neural Machine Translation](https://arxiv.org/abs/1606.02003). In Proceedings of EMNLP, 2016, pages 278–286. 3. Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio, [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473). arXiv preprint arXiv:1409.0473, 2014. From 617bc4e55a8464c25ccd01c77b3111dd177c9e78 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Thu, 13 Jul 2017 12:18:17 +0800 Subject: [PATCH 7/9] Refine README.md and refactor code structure for "machine translation with external memory" model. --- mt_with_external_memory/README.md | 190 +++++- mt_with_external_memory/data_utils.py | 15 + mt_with_external_memory/external_memory.py | 201 ++++++ .../image/lstm_c_state.png | Bin 0 -> 52288 bytes mt_with_external_memory/infer.py | 154 +++++ mt_with_external_memory/model.py | 206 ++++++ .../mt_with_external_memory.py | 598 ------------------ mt_with_external_memory/train.py | 157 +++++ 8 files changed, 891 insertions(+), 630 deletions(-) create mode 100644 mt_with_external_memory/data_utils.py create mode 100755 mt_with_external_memory/external_memory.py create mode 100644 mt_with_external_memory/image/lstm_c_state.png create mode 100644 mt_with_external_memory/infer.py create mode 100644 mt_with_external_memory/model.py delete mode 100644 mt_with_external_memory/mt_with_external_memory.py create mode 100644 mt_with_external_memory/train.py diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 583ab570bb..22c8effdcd 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -1,27 +1,29 @@ # 带外部记忆机制的神经机器翻译 -带**外部记忆**（External Memory）机制的神经机器翻译模型（Neural Machine Translation, NMT），是神经机器翻译模型的一个重要扩展。它利用可微分的外部记忆网络，来拓展神经翻译模型内部工作记忆（Working Memory）的容量或带宽，即引入一个高效的 “外部知识库”，辅助完成翻译等任务中信息的临时存取，有效改善模型表现。 +带**外部记忆**（External Memory）机制的神经机器翻译模型（Neural Machine Translation, NMT），是神经机器翻译模型的一个重要扩展。它引入可微分的记忆网络作为额外的记忆单元，拓展神经翻译模型内部工作记忆（Working Memory）的容量或带宽，辅助完成翻译等任务中信息的临时存取，改善模型表现。 -该模型不仅可应用于翻译任务，同时可广泛应用于其他需要 “大容量动态记忆” 的自然语言处理和生成任务，例如：机器阅读理解 / 问答、多轮对话、其他长文本生成等。同时，“记忆” 作为认知的重要部分之一，可用于强化其他多种机器学习模型的表现。 +类似模型不仅可应用于翻译任务，同时可广泛应用于其他需 “大容量动态记忆” 的任务，例如：机器阅读理解 / 问答、多轮对话、长文本生成等。同时，“记忆” 作为认知的重要部分之一，可用于强化其他多种机器学习模型的表现。 -本文所采用的外部记忆机制，主要指**神经图灵机** \[[1](#参考文献)\]，将于后文详细描述。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度学习的背景下，涌现出一系列有价值的工作，例如：记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。除神经图灵机外，其他均不在本文的讨论范围内。 +本文所采用的外部记忆机制，主要指**神经图灵机** \[[1](#参考文献)\] 方式（将于后文详细描述）。值得一提的是，神经图灵机仅仅是神经网络模拟记忆机制的尝试之一。记忆机制长久以来被广泛研究，近年来在深度学习的背景下，涌现出一系列有价值的工作，例如记忆网络（Memory Networks）、可微分神经计算机（Differentiable Neural Computers, DNC）等。本文仅讨论和实现神经图灵机机制。 -本文的实现主要参考论文\[[2](#参考文献)\]。本文假设读者已充分阅读并理解PaddlePaddle Book中[机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章。 +本文的实现主要参考论文\[[2](#参考文献)\]， 并假设读者已充分阅读并理解 PaddlePaddle Book 中 [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation) 一章。 ## 模型概述 ### 记忆机制简介 -记忆（Memory)，是人类认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（不同于感知）成为可能。记忆，同样是机器学习模型需要拥有的关键能力之一。 +记忆（Memory)，是认知的重要环节之一。记忆赋予认知在时间上的协调性，使得复杂认知（如推理、规划，不同于静态感知）成为可能。灵活的记忆机制，是机器模仿人类智能所需要拥有的关键能力之一。 -可以说，任何机器学习模型，原生就拥有一定的记忆能力：无论它是参数模型（模型参数即记忆），还是非参模型（样本即记忆）；无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络连接权值即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的；在预测时，模型是静态一致的，不拥有额外的跨时间步的信息记忆能力。 +#### 静态记忆 + +任何机器学习模型，原生就拥有一定的静态记忆能力：无论它是参数模型（模型参数即记忆），还是非参模型（样本即记忆）；无论是传统的 SVM（支持向量即记忆），还是神经网络模型（网络连接权值即记忆）。然而，这里的 “记忆” 绝大部分是指**静态记忆**，即在模型训练结束后，“记忆” 是固化的；在模型推断时，模型是静态一致的，不拥有额外的跨时间步的信息记忆能力。 #### 动态记忆 1 --- RNNs 中的隐状态向量 -当我们需要处理带时序的序列认知问题（如自然语言处理、序列决策优化等），我们需要在不同时间步上维持一个可持久的信息通路。带有隐状态向量 $h$（或 LSTM 中的细胞状态向量 $c$）的循环神经网络（Recurrent Neural Networks, RNNs） ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可叠加新的信息。这些信息在模型推断时随着不同的样本而不同，是 “动态” 的。 +在处理序列认知问题（如自然语言处理、序列决策等）时，由于每个时间步对信息的处理需要依赖其他时间步的信息，我们往往需要在不同时间步上维持一个持久的信息通路。带有隐状态向量 $h$（或 LSTM 中的状态 $c$）的循环神经网络（Recurrent Neural Networks, RNNs） ，即拥有这样的 “**动态记忆**” 能力。每一个时间步，模型均可从 $h$ 或 $c$ 中获取过去时间步的 “记忆” 信息，并可往上持续叠加新的信息以更新记忆。在模型推断时，不同的样本具有完全不同的一组记忆信息（$h$ 或 $c$），具有 “动态” 性。 -我们注意到，LSTM 中的细胞状态向量 $c$ 的引入，或者 GRU 中状态向量 $h$ 的以门（Gate）控制的线性跨层结构（Leaky Unit）的引入，从优化的角度看有着不同的理解：即为了梯度计算中各时间步的一阶偏导矩阵（雅克比矩阵）的谱分布更接近单位阵，以减轻长程梯度衰减问题，降低优化难度。但这不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅，如图1（引自[此文](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)）所示的 LSTM 中的细胞状态向量 $c$ 可视为这样一个用于信息持久化的 “线性记忆通道”。 +尽管上述对 LSTM中细胞状态 $c$ 的直觉说法有着诸多不严谨之处：例如从优化的角度看， $c$ 的引入或者 GRU 中的线性 Leaky 结构的引入，是为了在梯度计算中使得单步梯度的雅克比矩阵的谱分布更接近单位阵，以减轻长程梯度衰减问题，降低优化难度。但这不妨碍我们从直觉的角度将它理解为增加 “线性通路” 使得 “记忆通道” 更顺畅，如图1（引自[此文](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)）所示的 LSTM 中的细胞状态向量 $c$ 可视为这样一个用于信息持久化的 “线性记忆通道”。

@@ -30,37 +32,59 @@ #### 动态记忆 2 --- Seq2Seq 中的注意力机制 -然而这样的一个向量化的 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着一定程度的信息丢失。 +然而上节所属的单个向量 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着较大的潜在信息丢失。 -于是，注意力机制（Attention Mechanism）\[[3](#参考文献)\] 被提出，用于克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量，而是依赖一个向量组，向量组中的每个向量为编码器的各字符（Tokens）级编码向量（状态向量），并通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式提权信息用于序列的不同位置的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容获取不同的读取强度，起到一种和源语言 “软对齐（Soft Alignment）” 的作用。 +\[[3](#参考文献)\] 提出了注意力机制（Attention Mechanism），以克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量的信息，而是依赖一个向量组的记忆信息：向量组中的每个向量为编码器的各字符（Token）的编码向量（例如 $h_t$）。通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式读取信息，用于序列的不同时间步的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（请参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容决定不同的读取强度，起到一种和源语句 “软对齐（Soft Alignment）” 的作用。 -这里的 “向量组” 蕴含着更多更精准的信息，它可以被认为是一个无界的外部记忆模块（Unbounded External Memory）。“无界” 指的是向量组的向量个数非固定，而是随着源语言的字符数的变化而变化，数量不受限。在源语言的编码完成时，该外部存储即被初始化为各字符的状态向量，而在其后的整个解码过程中，只读不写（这是该机制不同于神经图灵机的地方之一）。同时，读取的过程仅采用基于内容的寻址（Content-based Addressing），而不使用基于位置的寻址（Location-based Addressing）。两种寻址方式不赘述，详见 \[[1](#参考文献)\]。当然，这两点局限不是非要如此，仅仅是传统的注意力机制如此，有待进一步的探索。 +相比上节的单个状态向量，这里的 “向量组” 蕴含着更多更精准的信息，例如它可以被认为是一个无界的外部记忆模块（Unbounded External Memory），有效拓宽记忆信息带宽。“无界” 指的是向量组的向量个数非固定，而是随着源语句的字符数的变化而变化，数量不受限。在源语句的编码完成时，该外部存储即被初始化为各字符的状态向量，而在其后的整个解码过程中被读取使用。 #### 动态记忆 3 --- 神经图灵机 -图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模型模拟可微分（于是可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式存储。神经图灵机正是要弥补这样的潜在缺陷。 +图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模拟可微分（即可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式的动态存储。神经图灵机正是要弥补这样的潜在缺陷。

图2. 图灵机结构漫画
-图灵机的存储机制，常被形象比喻成一个纸带（Tape），在这个纸带上有读头（Read Head）和 写头（Write Head）负责读出或者写入信息，纸袋的移动和读写头则受控制器 （Contoller) 控制（见图2，引自[此处](http://www.worldofcomputing.net/theory/turing-machine.html)）。神经图灵机则以矩阵$M \in \mathcal{R}^{n \times m}$模拟 “纸带”，其中 $n$ 为记忆向量（又成记忆槽）的数量，$m$为记忆向量的长度，以前馈神经网络或循环神经网络来模拟控制器，在 “纸带” 上实现基于内容和基于位置的寻址（寻址方式不赘述，请参考论文\[[1](#参考文献)\]），并最终写入或读出信息，供其他网络使用。神经图灵机结构示意图，见图3，引自\[[1](#参考文献)\]。 +图灵机的存储机制，常被形象比喻成在一个纸带（Tape）的读写操作。读头（Read Head）和 写头（Write Head）负责在纸带上读出或者写入信息；纸袋的移动、读写头的读写动作和内容，则受控制器 （Contoller) 控制（见图2，引自[此处](http://www.worldofcomputing.net/theory/turing-machine.html)）；同时纸带的长度通常有限。 + +神经图灵机则以矩阵 $M \in \mathcal{R}^{n \times m}$ 模拟 “纸带”，其中 $n$ 为记忆向量（又成记忆槽）的数量，$m$ 为记忆向量的长度。以前馈神经网络或循环神经网络来模拟控制器，决定本次读写在不同的记忆槽上的读写强度分布，即寻址： + + - 基于内容的寻址（Content-based Addressing)：寻址强度依赖于记忆槽的内容和该次读写的实际内容； + - 基于位置的寻址(Location-based Addressing)：寻址强度依赖于上次寻址操作的寻址强度（例如偏移）； + - 混合寻址：混合上述寻址方式（例如线性插值）； + +（详情请参考论文\[[1](#参考文献)\]）。根据寻址情况，图灵机写入 $M$ 或从 $M$ 读出信息，供其他网络使用。神经图灵机结构示意图，见图3，引自\[[1](#参考文献)\]。

图3. 神经图灵机结构示意图
-和上述的注意力机制相比，神经图灵机有着诸多相同点和不同点。相同在于：均利用矩阵（或向量组）形式的存储，可微分的寻址方式。不同在于：神经图灵机有读有写（是真正意义上的存储器），并且其寻址不仅限于基于内容的寻址，同时结合基于位置的寻址（使得例如 “长序列复制” 等需要 “连续寻址” 的任务更容易），此外它是有界的（Bounded)；而注意机制仅仅有读操作，无写操作，并且仅基于内容寻址，此外它是无界的（Unbounded）。 +和上节的注意力机制相比，神经图灵机有着诸多相同点和不同点。相同点例如： + +- 均利用矩阵（或向量组）形式的外部存储。 +- 均利用可微分的寻址方式。 + +不同在于： + +- 神经图灵机有读有写，是真正意义上的存储器；而注意力机制在编码完成时即初始化存储内容（仅简单缓存，非可微分的写操作），在其后的解码过程中只读不写。 +- 神经图灵机不仅有基于内容的寻址，同时结合基于位置的寻址，使得例如 “序列复制” 等需 “连续寻址” 的任务更容易；而注意力机制仅考虑基于内容的寻址，以实现 Soft Aligment。 +- 神经图灵机利用有界（Bounded) 存储；而注意力机制利用无界（Unbounded）存储。 + +#### 三种记忆方式的混合，以强化神经机器翻译模型 -#### 三种记忆混合，强化神经机器翻译模型 +尽管在一般的序列到序列模型中，注意力机制已经是标配。然而，注意机制中的外部存储仅用于存储编码器信息。在解码器内部，信息通路仍依赖 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充解码器内部的单向量信息通路，成为自然而然的想法。 -尽管在一般的序列到序列模型中，注意力机制已经是标配。然而，注意机制的外部存储仅仅是用于存储源语言的字符级信息。在解码器内部，信息通路仍然是依赖于 RNN 的状态单向量 $h$ 或 $c$。于是，利用神经图灵机的外部存储机制，来补充解码器内部的单向量信息通路，成为自然而然的想法。 +于是，我们混合上述的三种动态记忆机制，即RNN 原有的状态向量、注意力机制被保留；同时，基于简化版的神经图灵机的有界外部记忆机制被引入以补充解码器单状态向量记忆。整体的模型实现参考论文\[[2](#参考文献)\]。少量的实现差异，详见[其他讨论](#其他讨论)一章。 + +这里额外需要理解的是，为什么不直接通过增加 $h$ 或 $c$的维度来扩大信息带宽？ + +- 一方面因为通过增加 $h$ 或 $c$的维度是以 $O(n^2)$ 的存储和计算复杂度为代价（状态-状态转移矩阵）；而基于神经图灵机的记忆扩展代价是 $O(n)$的，因其寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。 +- 基于状态单向量的记忆读写机制，仅有唯一的读写强度，即本质上是**全局**的；而神经图灵机的机制是**局部**的，即读写本质上仅在部分记忆槽（寻址强度的分布锐利，即真正大的强度仅分布于部分记忆槽）。局部的特性让记忆的存取更干净，干扰更小。 -当然，我们也可以仅仅通过扩大 $h$ 或 $c$的维度来扩大信息带宽，然而，这样的扩展是以 $O(n^2)$ 的存储和计算复杂度为代价（状态-状态转移矩阵）。而基于神经图灵机的记忆扩展代价是 $O(n)$的，因为寻址是以记忆槽（Memory Slot）为单位，而控制器的参数结构仅仅是和 $m$（记忆槽的大小）有关。另外值得注意的是，尽管矩阵拉长了也是向量，但基于状态单向量的记忆读取和写入机制，本质上是**全局**的；而神经图灵机的机制是局部的，即读取和写入本质上只在部分记忆槽（尽管实际上是全局写入，但是寻址强度的分布是很锐利的，即真正大的强度仅分布于部分记忆槽），因而可以认为是**局部**的。局部的特性让记忆的存取更干净，干扰更小。 -所以，在该示例的实现中，RNN 原有的状态向量和注意力机制被保留；同时，基于简化版的神经图灵机的有界外部记忆机制被引入以补充解码器单状态向量记忆。整体的模型实现参考论文\[[2](#参考文献)\]，但有少量差异，详见[其他讨论](#其他讨论)一章。 ### 模型网络结构 @@ -79,39 +103,141 @@ 1. $M_{t-1}^B$ 和 $M_t^B$ 为有界外部存储矩阵，前者为上一时间步存储矩阵的状态，后者为当前时间步的状态。$\textrm{read}^B$ 和 $\textrm{write}$ 为对应的读写头（包含其控制器）。$r_t$ 为对应的读出向量。 2. $M^S$ 为无界外部存储矩阵，$\textrm{read}^S$ 为对应的读头，二者配合即实现传统的注意力机制。$c_t$ 为对应的读出向量。 -3. $y_{t-1}$ 为解码器上一步的输出字符并做词向量（Word Embedding)，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 +3. $y_{t-1}$ 为解码器上一步的输出字符并做词向量（Word Embedding）映射，作为当前步的输入，$y_t$ 为解码器当前步的解码符号的概率分布。 4. 虚线框内（除$M^S$外），整体可视为有界外部存储模块。可以看到，除去该部分，网络结构和 RNNsearch\[[3](#参考文献)\] 基本一致（略有不一致之处为：用于 attention 的 decoder state 被改进，即叠加了一隐层并引入了 $y_{t-1}$）。 ## 算法实现 -算法实现的关键部分在辅助类`ExternalMemory` 和模型函数 `memory_enhanced_seq2seq`。 +算法实现于以下几个文件中： + +- `external_memory.py`: 主要实现简化版的 **神经图灵机** 于 `ExternalMemory` 类，对外提供初始化和读写函数。 +- `model.py`: 相关模型配置函数，包括双向 GPU 编码器（`bidirectional_gru_encoder`），带外部记忆强化的解码器（`memory_enhanced_decoder`），带外部记忆强化的序列到序列模型（`memory_enhanced_decoder`）。 +- `data_utils.py`: 相关数据处理辅助函数。 +- `train.py`: 模型训练。 +- `infer.py`: 部分示例样本的翻译（模型推断）。 ### `ExternalMemory` 类 `ExternalMemory` 类实现通用的简化版**神经图灵机**。相比完整版神经图灵机，该类仅实现了基于内容的寻址（Content Addressing, Interpolation），不包括基于位置的寻址（ Convolutional Shift, Sharpening)。读者可以自行将其补充成为一个完整的神经图灵机。 -类结构如下： +该类结构如下： ``` class ExternalMemory(object): - __init__(self, name, mem_slot_size, boot_layer, readonly, enable_projection) - __content_addressing__(self, key_vector) - __interpolation__(self, head_name, key_vector, addressing_weight) - __get_adressing_weight__(self, head_name, key_vector) - write(self, write_key) - read(self, read_key) + """External neural memory class. + + A simplified Neural Turing Machines (NTM) with only content-based + addressing (including content addressing and interpolation, but excluding + convolutional shift and sharpening). It serves as an external differential + memory bank, with differential write/read head controllers to store + and read information dynamically as needed. Simple feedforward networks are + used as the write/read head controllers. + + For more details, please refer to + `Neural Turing Machines `_. + """ + + def __init__(self, + name, + mem_slot_size, + boot_layer, + readonly=False, + enable_interpolation=True): + """ Initialization. + + :param name: Memory name. + :type name: basestring + :param mem_slot_size: Size of memory slot/vector. + :type mem_slot_size: int + :param boot_layer: Boot layer for initializing the external memory. The + sequence layer has sequence length indicating the number + of memory slots, and size as memory slot size. + :type boot_layer: LayerOutput + :param readonly: If true, the memory is read-only, and write function cannot + be called. Default is false. + :type readonly: bool + :param enable_interpolation: If set true, the read/write addressing weights + will be interpolated with the weights in the + last step, with the affine coefficients being + a learnable gate function. + :type enable_interpolation: bool + """ + + def _content_addressing(self, key_vector): + """Get write/read head's addressing weights via content-based addressing. + """ + pass + + def _interpolation(self, head_name, key_vector, addressing_weight): + """Interpolate between previous and current addressing weights. + """ + pass + + def _get_addressing_weight(self, head_name, key_vector): + """Get final addressing weights for read/write heads, including content + addressing and interpolation. + """ + pass + + def write(self, write_key): + """Write onto the external memory. + It cannot be called if "readonly" set True. + + :param write_key: Key vector for write heads to generate writing + content and addressing signals. + :type write_key: LayerOutput + pass + + def read(self, read_key): + """Read from the external memory. + + :param write_key: Key vector for read head to generate addressing + signals. + :type write_key: LayerOutput + :return: Content (vector) read from external memory. + :rtype: LayerOutput + """ + pass ``` -神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，注意这里的`is_seq`需设成`True`，该序列的长度表示记忆槽的数量，`size` 表示记忆槽（向量）的大小。同时依赖一个外部层作为初始化， 记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 +其中，私有方法包含： + +- `_content_addressing`: 通过基于内容的寻址，计算得到读写操作的寻址强度。 +- `_interpolation`: 通过插值寻址（当前寻址强度和上一时间步寻址强度的线性加权），更新当前寻址强度。 +- `_get_addressing_weight`: 调用上述两个寻址操作，获得对存储导员的读写操作的最终寻址强度。 + + +对外接口包含： + +- `__init__`：类实例初始化。 + - 输入参数 `name`: 外部记忆单元名，不同实例的相同命名将共享同一外部记忆单元。 + - 输入参数 `mem_slot_size`: 单个记忆槽（向量）的维度。 + - 输入参数 `boot_layer`: 用于内存槽初始化的层。需为序列类型，序列长度表明记忆槽的数量。 + - 输入参数 `readonly`: 是否打开只读模式（例如打开只读模式，该实例可用于注意力机制）。打开是，`write` 方法不可被调用。 + - 输入参数 `enable_interpolation`: 是否允许插值寻址（例如当用于注意力机制时，需要关闭插值寻址）。 +- `write`: 写操作。 + - 输入参数 `write_key`：某层的输出，其包含的信息用于写头的寻址和实际写入信息的生成。 +- `read`: 读操作。 + - 输入参数 `read_key`：某层的输出，其包含的信息用于读头的寻址。 + - 返回：读出的信息（可直接作为其他层的输入）。 -`ExternalMemory`类的寻址逻辑通过 `__content_addressing__` 和 `__interpolation__` 两个私有函数实现。读和写操作通过 `read` 和 `write` 两个函数实现。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 +部分重要的实现逻辑： -为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中，并且采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 +- 神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，并采用序列形式（`is_seq=True`），该序列的长度表示记忆槽的数量，序列的 `size` 表示记忆槽（向量）的大小。该序列依赖一个外部层作为初始化， 其记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 -`ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。便于用该类等价实现传统的注意力机制。 + ``` + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) + ``` +- `ExternalMemory`类的寻址逻辑通过 `_content_addressing` 和 `_interpolation` 两个私有方法实现。读和写操作通过 `read` 和 `write` 两个函数实现，包括上述的寻址操作。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 +- 为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中。控制器主要包括寻址操作和写操作时生成写入/擦除向量等，其中寻址操作通过上述的`_content_addressing` 和 `_interpolation` 两个私有方法实现，写操作时的写入/擦除向量的生成则在 `write` 方法中实现。上述均采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 +- `ExternalMemory` 类具有只读模式，同时差值寻址操作可关闭。主要目的是便于用该类等价实现传统的注意力机制。 -注意， `ExternalMemory` 只能和 `paddle.layer.recurrent_group`配合使用，具体在用户自定义的 `step` 函数中使用，它不可以单独存在。 +- `ExternalMemory` 只能和 `paddle.layer.recurrent_group`配合使用，具体在用户自定义的 `step` 函数中使用（示例请详细代码），它不可以单独存在。 ### `memory_enhanced_seq2seq` 及相关函数 diff --git a/mt_with_external_memory/data_utils.py b/mt_with_external_memory/data_utils.py new file mode 100644 index 0000000000..6cc481fef6 --- /dev/null +++ b/mt_with_external_memory/data_utils.py @@ -0,0 +1,15 @@ +""" + Contains data utilities. +""" + + +def reader_append_wrapper(reader, append_tuple): + """ + Data reader wrapper for appending extra data to exisiting reader. + """ + + def new_reader(): + for ins in reader(): + yield ins + append_tuple + + return new_reader diff --git a/mt_with_external_memory/external_memory.py b/mt_with_external_memory/external_memory.py new file mode 100755 index 0000000000..d3dbf32ff3 --- /dev/null +++ b/mt_with_external_memory/external_memory.py @@ -0,0 +1,201 @@ +""" + External neural memory class. +""" +import paddle.v2 as paddle + + +class ExternalMemory(object): + """ + External neural memory class. + + A simplified Neural Turing Machines (NTM) with only content-based + addressing (including content addressing and interpolation, but excluding + convolutional shift and sharpening). It serves as an external differential + memory bank, with differential write/read head controllers to store + and read information dynamically. Simple feedforward networks are + used as the write/read head controllers. + + The ExternalMemory class could be utilized by many neural network structures + to easily expand their memory bandwidth and accomplish a long-term memory + handling. Besides, some existing mechanism can be realized directly with + the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an + unbounded external memory). + + Besides, the ExternalMemory class must be used together with + paddle.layer.recurrent_group (within its step function). It can never be + used in a standalone manner. + + For more details, please refer to + `Neural Turing Machines `_. + + :param name: Memory name. + :type name: basestring + :param mem_slot_size: Size of memory slot/vector. + :type mem_slot_size: int + :param boot_layer: Boot layer for initializing the external memory. The + sequence layer has sequence length indicating the number + of memory slots, and size as memory slot size. + :type boot_layer: LayerOutput + :param readonly: If true, the memory is read-only, and write function cannot + be called. Default is false. + :type readonly: bool + :param enable_interpolation: If set true, the read/write addressing weights + will be interpolated with the weights in the + last step, with the affine coefficients being + a learnable gate function. + :type enable_interpolation: bool + """ + + def __init__(self, + name, + mem_slot_size, + boot_layer, + readonly=False, + enable_interpolation=True): + self.name = name + self.mem_slot_size = mem_slot_size + self.readonly = readonly + self.enable_interpolation = enable_interpolation + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) + # prepare a constant (zero) intializer for addressing weights + self.zero_addressing_init = paddle.layer.slope_intercept( + input=paddle.layer.fc(input=boot_layer, size=1), + slope=0.0, + intercept=0.0) + # set memory to constant when readonly=True + if self.readonly: + self.updated_external_memory = paddle.layer.mixed( + name=self.name, + input=[ + paddle.layer.identity_projection(input=self.external_memory) + ], + size=self.mem_slot_size) + + def _content_addressing(self, key_vector): + """ + Get write/read head's addressing weights via content-based addressing. + """ + # content-based addressing: a=tanh(W*M + U*key) + key_projection = paddle.layer.fc( + input=key_vector, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + key_proj_expanded = paddle.layer.expand( + input=key_projection, expand_as=self.external_memory) + memory_projection = paddle.layer.fc( + input=self.external_memory, + size=self.mem_slot_size, + act=paddle.activation.Linear(), + bias_attr=False) + merged_projection = paddle.layer.addto( + input=[key_proj_expanded, memory_projection], + act=paddle.activation.Tanh()) + # softmax addressing weight: w=softmax(v^T a) + addressing_weight = paddle.layer.fc( + input=merged_projection, + size=1, + act=paddle.activation.SequenceSoftmax(), + bias_attr=False) + return addressing_weight + + def _interpolation(self, head_name, key_vector, addressing_weight): + """ + Interpolate between previous and current addressing weights. + """ + # prepare interpolation scalar gate: g=sigmoid(W*key) + gate = paddle.layer.fc( + input=key_vector, + size=1, + act=paddle.activation.Sigmoid(), + bias_attr=False) + # interpolation: w_t = g*w_t+(1-g)*w_{t-1} + last_addressing_weight = paddle.layer.memory( + name=self.name + "_addressing_weight_" + head_name, + size=1, + is_seq=True, + boot_layer=self.zero_addressing_init) + interpolated_weight = paddle.layer.interpolation( + name=self.name + "_addressing_weight_" + head_name, + input=[addressing_weight, addressing_weight], + weight=paddle.layer.expand(input=gate, expand_as=addressing_weight)) + return interpolated_weight + + def _get_addressing_weight(self, head_name, key_vector): + """ + Get final addressing weights for read/write heads, including content + addressing and interpolation. + """ + # current content-based addressing + addressing_weight = self._content_addressing(key_vector) + # interpolation with previous addresing weight + if self.enable_interpolation: + return self._interpolation(head_name, key_vector, addressing_weight) + else: + return addressing_weight + + def write(self, write_key): + """ + Write onto the external memory. + It cannot be called if "readonly" set True. + + :param write_key: Key vector for write heads to generate writing + content and addressing signals. + :type write_key: LayerOutput + """ + # check readonly + if self.readonly: + raise ValueError("ExternalMemory with readonly=True cannot write.") + # get addressing weight for write head + write_weight = self._get_addressing_weight("write_head", write_key) + # prepare add_vector and erase_vector + erase_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + add_vector = paddle.layer.fc( + input=write_key, + size=self.mem_slot_size, + act=paddle.activation.Sigmoid(), + bias_attr=False) + erase_vector_expand = paddle.layer.expand( + input=erase_vector, expand_as=self.external_memory) + add_vector_expand = paddle.layer.expand( + input=add_vector, expand_as=self.external_memory) + # prepare scaled add part and erase part + scaled_erase_vector_expand = paddle.layer.scaling( + weight=write_weight, input=erase_vector_expand) + erase_memory_part = paddle.layer.mixed( + input=paddle.layer.dotmul_operator( + a=self.external_memory, + b=scaled_erase_vector_expand, + scale=-1.0)) + add_memory_part = paddle.layer.scaling( + weight=write_weight, input=add_vector_expand) + # update external memory + self.updated_external_memory = paddle.layer.addto( + input=[self.external_memory, add_memory_part, erase_memory_part], + name=self.name) + + def read(self, read_key): + """ + Read from the external memory. + + :param write_key: Key vector for read head to generate addressing + signals. + :type write_key: LayerOutput + :return: Content (vector) read from external memory. + :rtype: LayerOutput + """ + # get addressing weight for write head + read_weight = self._get_addressing_weight("read_head", read_key) + # read content from external memory + scaled = paddle.layer.scaling( + weight=read_weight, input=self.updated_external_memory) + return paddle.layer.pooling( + input=scaled, pooling_type=paddle.pooling.Sum()) diff --git a/mt_with_external_memory/image/lstm_c_state.png b/mt_with_external_memory/image/lstm_c_state.png new file mode 100644 index 0000000000000000000000000000000000000000..ce791573f359a2e3a346eb920b1edec6983cc184 GIT binary patch literal 52288 zcmd43XIK;47dMK>f`W)1K$?IeRjME$T|_{-h}2M}NtYU02sTivbm_e}>7A%Z?=?Vx zNG}0G4hotW$;imr z$;keCboC1GM8|1o2l#QxsaBM&k%)?cJQ7hHzDnjz-Wi^o0 zX!N6l9gjf_Mg=kK7Rxm-u!Y*l8TTK;&c|NTYD1$jxXSQ{F*S*~>Kxl#yu1K$|6c3q zHM#U0ruR1N`xvSdb3-_%@iYAY{gRAKQGvrg(k9&Z#b_IT4j}ixOR-a4z)B;{Fp}=S zCuC1Ha)C!4(E7jr_vpv)Q`6`FyCdUNxq0n>cbUsq*)RO>F7A#3$Nz4EV(x#wbbc4a zQN(LbPpSEi07!r=?jr(8_0K4VhVZ8}#?gOCKf25(@{W$T=C!x)AEUiO7|1~FT@M0X z9w;qdCFLBYr8RH}_6Ob(KDqVmAHry(qecgNXWv%|U$+jY{S^HrT!%Jza@R74qIQ1y z9^K?bE&pdaipymV#;X&*yOtr+jK|;<(hLp%?+jrxsliyW&|kw0Jt86r9KMl!uwtj@ zpFEQ?JIY+m7F1u}V|$~7_;vETmqiypCkVWK-Q`mBKcp+gHQ(PKQ}Nf4>z{EP?^u({AtF%u{*s>%hAo?sRnh!?Dos)wRK`3%|>s8{u6VTqP4p9ZGf@BuRjt{;LhcUnWgrnU$kCdJgdSblIxPPs@B5TuyUJT}ltC zbV?B9n!xZMHx>UI{%-h^zIdVoTAnvdFZkt1yasvvtKoI=nsG3%2|w*$Wp9ia5;a^U zD1kK;um7u7PsrVMN5~D;0Ev{{bhbNT#O1o&l z5s0a(P8X_8Y12-Uc2CjvIh-?s%=0dDs{z|62H-ZkiLT+_L(0x(ZC`8mPv3L&xUs}{ z^yV0E*%r(~PG)1-bVqz!O@}Eq?|P`G2noZN0BL=y zER(D&D;sd=7eEwS0i6Ht{QAEMZ#kkA^AUBU`dCXQW60Gjy058|GjL^H$9E`HDbI->*miLK1?i{M z=ljo;cQ#~u88c&yc%^5?tWoR-#kSsj8N(mn!S_`40qwrp3h2u6546cQZ+&qAjGt$u ziB`$X?N0RxF5va+&i`qkl68(a7y@~&p75ZrB*%>oQ4qH6SXu~4u#}s^GKRF;8y$f5 zc(zWTbz)cJUAP=rzgqGE6I{9d{vRj_VxMu>9K>S3wxR8ReP?7Wd2SG@W4*GUbWG*O zean00Oj$Xpa9ZG-Qw3(IW);<%KY68qc8%iwi@p^4&WQd5FaYB}#3TOIZVQ?$Uc>1# z|D$^)d?6v}rZN`%&y1wauX2>7NgCZ|p3J*jB?l93%(V}6_r*r+57YXUMRoF2C5Q`t zPWgK^ct9Y+TRkw(yh#+$Q41dSe{pN-nV^FQWn=BpJ?B1sn-;G1H^>%MY0G=UG0Rfr zU!+$+*V!+|)}vAkngj&O)bd0qG>0Ag8Yc@9u^Sr2xoZ-gh2{2cr4k4L?Sg(r{f|`1 zvDH-yK)%>?04+u=ZzkPHo_qS@J4eqE3!iPG^TW^mN@ZGd_2Mmu5bZV=^eBJt4u!^i zxuXd!D#D{+l(*zwGsish8i@hJPyRKh?_VaJa+u3Xs5eg2U$gM{F(v(Q=qYu}@j|6{ z;&a{jg=R)i;lYW%>M^~q8K$y@tVOaf6x31u@U zCmM~_ggdg;HG2a-wDIDz{P|hF@w%~hw&NScIPE2X<$`bgr*l-+IrYQpe?X@t8FGIo|6A zX+}66NW;Az$(SW`8mGAP;aBVO<%~qcuHP4MHDXB7*VX%O7(l`-=DYtX(GyVlbVNU( z>E&lPHWgR1_3OV<&Dd$_bF0)1U3KcM*gt4&DRzVlPDtfD`Psgj_BK*APBV=&YT;q7pef(40;z~`7g$y zO3SE|Trv_2Su3`_%UKJmNjCn%CdH=ztz~n!EKO&6a?+`b<{D>O@$qzA-5smGheak~ zR@Tdn>AVNQm+Mv-5bkP!KUEvuJZuI$gl=+Cys34#j_J~(;TwuWF#xGDS*Sxuk|ZNT zEoY0@zqlH~F`Y1G55Pg45p2Ae?O++vBP^0t%LRLs-n05B`?bb=Osh_G=a1#>BQZ|X zq1xBpN+Z(=pb`srZ_}l}VR36)#Tr~WfLnn`(a-?XUS$3k@_$H8^QOp38vEUvTm4BTYlxw@}fJelfSvFC1kniYnwiu&18zQh9yNNK<145 zQszTL6hbt2yh@l7{Tq2ExNCWVEeP-a7YLLt^@j2H#W)*1P>)lo)O!CEsTFMHSg>r; zrNWb*9R% zVwS~h9bE^kkO}?YcC30{YPy@WNq7BDt$J(iJAKw)V#EbA-TXmU&HC$~<14axM)&hS?b)}NTm)$mxpRP-*Cj44Uic?Kv?%OT z;^pQ6lna2F$)Qw@kcj!sRXQ^UWv)6pQFg$Z`k3fD)#m2Y->BCbM>MWnEY>8=;Qyc3 zfc)a2fS1oO9ZUv)oi3a(O){e$vN>Qfm6Wst&1EPwR5|O&Bo2AC1v{~cDjScSl(T)i z1JMSiKnSy50fYuz*eCsh)H29wexqI$$!0+bSb*vQswW{q-%(@x5}9Gzt)lY^j=#J<9OpcXtX!*b|&%WzX9aog|7YnxCQBazP=D8X8~tpJ=o|CicSp% zIWdU`l4AG0pf8v9SL>G+nv4=Ij%?;jNif$o;OA}tlN?g_F^tV3BS_=`kwwBKE>{(< zngHorF58mE2S@WDuL=&0dU}s-kXU=Lif%F30g;?Z;@}77`=jUYkYT?$p6Y13OWx0+ z(^L-tnT4!9oN<3@uVLY8Us3DDS3f=#IW#&m>6G_?sa2#t|9b~%_ZH7B|BVlR+_hE{ zrgIORJFqDV2pgN0LqyF9n66S$`i69-^hg0YSfw=W# z$XAj^WMXoSMJd_~#~{6Yu~plVACwX9n!ie}Lbz{rq&{1o-s`S@on_@K(+w=^wDj#3 z5Oxf!hqJF_$E7@Y)KnBu9ZNZ;WgQ;=@NZF;3uHgYhT|EdEopndMSkdhk)?OleH)^Y zVAE6OEScc=+C^p`m=f`YXNRM`LHCw%mG4a;hMQ78H}kj`a=ni?GJ90%lIJwCCa8Wr z>30CKBE;WT;TjANjG13Zy{x{kTE*WOS>j&tfCuA9sR}l<=vz<#D6+U}Wv>rxnpHTo z6i7_!=$Pg_vpNJWNp4F0MHZxRKX#CpLK^C=8rNkF$}7LnC?3au5OSF=(D~lRtIKst zJirvfZW^-F8gb`G$P|nS3X(3BLlLy>kb$1f1dryi403w_~O<`;reX;?Zx;#T3=JjS@LXCgWrG^(A?WSx@oK4yeF(9pQAxM*gDa9;$bmUqK7li>HF5x`KY zX`4Xx@_Eq_ea}Lp1gEZqrN{0piEnQK6!HRcal0cKS>1Cxheu@lf(@;5=&$e0cH+Iu zy2qox&Yq1{?5c2ZFaLe~y^M_t2nT}#6-W{6i;p_iWxf2N5;ppNot_2HqB;92m!%@_ zbj3T4mh@&{REaE|vbm z1q{dDB5~vgm%C-JQMw;rG+%CWD*H_2{h@=JG>-UEXJSR49>p)2ejj;I zbPCwiu%e@)s&+Z5=q4@g@zH>jjeRg?3G}(@{k)cnV+eJWm_nywUzXp|&NyP_s_5cJGUk zx=aQz$@NG7Ok!jGD#M7@wYzjz8nqn50YA8U7x*pq^O~=_sllhG3JNzp8J;uI6v@^0 z4+ZnuX}J4!R<;ZlyaWs@Q(LDWAloSx$g>OPHN@0~Y&cu=H4JzCC6Q7uvPd z|NatSId^4-bY*TO+IY>T(J5yo)V3vcE7>c@kakY?&(5nJRP)S#Ga9kHYLBQ}{L&y( z5t~Y?kKkY9tCQz?Lg_3GzbDkSrSpHggEu2R4<{D^FmT{aw_rq>st`tcW~ZTi@ykc2 z6cv(K-}P4UcPzOzrd_|ocS+LG2*unpe2L$cB(JD-8$f{&AH^D12%F(!4h6LSvz#7(96$bVd%T4U~E4Dx$D5PaZ?fwCl2~O1* z-h5pLoq-UWO?Ux%V663Ie;Zz>v3uu zsaif&I#30Eb^TuClBc0im81dM_E$o-*KdHr7smgv$SBt6b{~0C_R!4~P6^#45BwmL1UX?O2lM0WaE#;Ih-(s{go!#9quWt3z_f^}NQY1Lt8iK& zcF978IbzWMP zjL|XfpTg59b+!6=DRB*|oOIt((iE7KE&Isk?hrg;T-kiormD#_E;h2al9I4qZ zSkECx+p&23x)li0`(TdJ?guD%YO#MTxt3viJL$!m>G4EihCFa$z>vM31gOjhGY^+q z?xcZv@EwDMi48TB7{|5=tr&#Q>I%g!J@(r?3K@vh zwY7=W(iR|!~7&0?3@js&JP0bjjC1^PCnQtV<$qNtN}*|Yl{5XZ6$v33=N@;+R< zpAkyjDVOi76EOAOmce7;ut6vJz%?z!ZYtBjQ@Uxfk`&tVl*`Situ)#w&ih^N63|Q_ z9^iR*f%p~B>)|A)Zx66CR#fj++JOaL7_$gGWd3-1!PM|9LP@CO#w^YPeQIY01Uyaq zVjJzQ1i&X@}9ck$oJ4mA|322k!vMOFyOC>^GSO%Hu-dBz_C z82`lHNPn;8;phE}#|xEhwS)Nntghb)tsrhysRh2S_1`qwJaPULAiXiSsA>NzG1iC# zTeR31$6W6P7>59$nk`qh6ah#1#DMmf#>i3ad9e7Mx34Of-O^f)>6<9~2Od`zOyg1v z(ci0$!@l=W6}zcakQCjcI$b8a=B-;7#$mr1&RfhX*B`3O1~|V%cE5EXR?57s{AoZs zg1@AFWapf@bpBf58gG;+$Y(Du*CLxmPS-C&6KlM$EUm!JGwpM{dkw4VoAQ^u5oJ{D zb~}~Te^*v@=gR^1x~_DNoG&OfYo`4_eg!65|7^NtG{SKCqLcn|Reac-qadsjauk&l zC2vB0XLsT5!mHyqSFC8uAc6DFLuq=91?SV4{xeOO#&@HT=)4OPA%Lp`3Mn5H96#|$ zWMX>+n@-+8Q}=Mhp_bY_bBlPHQ_vf}qDkQ^O_9g5U~s&DElAG}*kESKEWrh!;)(XN zP;qMZ9+LkVeix}f#PD2UImRYzTN>)C?XL(`WWt80n@0G@Ok)FpCF<1@z(^~)&q(9u3x*Ye_sk=J-RlbA3j*} zj1L;lBo^8Pa3E7#c#GshfeR6yEym9INyqZXrD)8&hdFNf0;2Bli=w2zU0k+L(b_eb^ONPbuCrxLv9)MCVLL9W1b zxC1G$t;@{GGy1?@+l6cIjpr+)Plq(N3RrbU>DzcUt{7Ji1hN5x7}iNzv#!#}vy%Jk z3RF)w4Y#E$v_AJqy^J#~v8b`qh^8PMe#lbi;f=7etMImtcyN~dG+tT|Wmq1ad%4d% zjJQMa$7{3URH-mrJynK){s86}6PcdUUMPH*wGbKL-&VbV>e1!y^)hKpK-wvg&S@MQ z9@zGX$M3AC>Sc3Ul7bCVsW~X{CV-#WUJ7Rf`LiDUQY$rzEea1hH8Hd5j2Qi>x->B5 zW_5VduDziGC;Zx%G2qU?`=-6!tTd|=UkU#*@Vv~)cr|cyDPvFG=LhS4nvZn_B{v`% zGSBjM#^{UI_LE9s^e(iYwbG>&9=aY(=+x;0ke~2$fNl0{uBy(n8q$9)BSC_nl{0Pg zX;C>4qt*$K6z`BXXVpS!s~$7Y(tLJ?s&!ILa|kv-o#;kBEJzJ-51bstEj#zwQ`5Bz zkLN_mZKj{((flnWhbum61(nydTUt(6X2|sf;%gX8%+I^ zM{n*Ty{wzT-+cBSwfzb-6S(<>*zN7KUDs94weac^e%B44Ea<1>!JU^E^T?JyFWi@4 zdh^(3lcNJ~9hG8mb$`YKj(-hnBX%*#Iy;0cEV2>L$hBj@fgREc8X@@|4K0ut2OaF}A zhi#<1W_xj8**DGe8<=}zFE$@MWd zX~Bv3o9?Zq-qV~?`-MX_V1_Lzf9?*ep6)CX=#UKyTzpM6d1I?)Oe+f|wOxVq^@tYv zRL6R>z{ofka4^)O&Fc2mfv?l4hAAvsN8G_p)Uq1!&3L12ceao9uU+T=dI1tR2IlSZ zUk9@HejEUfzr63$+*n2c7J7xp7C@2Vl5m&iwbiPMw16y|a2-Ixb9-GW18ve!_FAzE zi}Z5P0K&Zgjt8gx>Q>sTTRG>X3CI0@{ZKPSZlYxmo07oMDhthWKP`3cWj6vz7W~0b zw0g6eLV)9G&f@uK31UZnxP=+ zUy;PMpESgq@z{6{$l)Kn*Yb$UN4ojXE@0MD39rrZZnrxT8ISbucilMm z9%RCoASh6x9h4Y?Y<7j!{s<&&B#IG_>f@(mZ$)wTJ%#T#o6KjG+ebs!e};6)L-(Vx zhxb(MVh6tu1VVlOXjA7+5{VEDvN>QPbmDlgAW+{t1+4tI?F?U}i6IAk+6@PiAk_z( z-h)7~RG~F@KQf6ZOHJSCk(3?MRkCTceM$1LWDHe{dkulUC}&1QUA&f^=ZFzf zUlH;_FT*_Ux~64VEt7{%udmxk>`DmRe$vTStDVD+?}57>p-{{2(W<+oEM_aLj$>fP zKmWnf#C1PF(`A(bLwO?Pv!6r^!yg|b>%lhz7UFWmtTvB7?Sz}{JO??Z*zQ!u26H;B zlVFD|xDbMwlSq|tbuo(}V0wWjl^#s3!x4qQ@9I$~fOGsJKb7BETR-Jvu5Ii?%&VEX zcYLy8OM`rCMXDS>gquK|ZG8fsx~Ri(Yp`@rkLlD0B@b$r2BjI0f(5KVfoA<-g6(Cp zO#1#5|97x9!QAaBsk3VGk_U2rC5hIiLku|;f!kYWc2P2&uY9cvM}Rnp?7F3aIYiUD zqV94}r?oMCW)63ouCT#KBfUJtcU=@{1 zDk$x!Ihw87gAh?uQ>7KxDEU>@Q5XNMQp5iG*WTylu=R(eWKwMldSLF6ak}QQofC_h zhSqUuroUO_EL?kIlu!=a33k#R0wPN?i?@%@cD2dLSx=(t8V?2|kjH)IzH1o-0aM5x zrgrX7HCe_h8<1S;DG{O`HgA;8tINGva-bEaC3|}4lw>ebhLlhDS!9a@5D8NbAd;fG zxXoQad!tA(!;9D8c}O#C0i_rvWmvnClhk^Zv}VaGef{K&xwC`*DCl8qKa_|wUpu8I zF14I2QtphdsBiCCaet%UX>~oUjAEnRX*)>2jBsyaxm#XFefnNNwHlLN6iT(Js7z6| z`YY715tM={^@&o~1fblRl4Qr6$IbqMXr~ju3hHf_*IF^=Q=FH%T*&w0V$!cD2MFl5*@K{@LGPToZjYtw z-vI@jdyN4DjAKxHSrkXIDnz19Sy9Sa#0xaT<=<=kDaBfJ1mWFy_w)o2A8!COA zZ>;xInyHEwY^W?|T)i~E^S12CxNA6X&Mc_`NB9N5QPp@%F7Ft3kDC)Cjr~UQr^+!c z$v)W5s;j-m%JS?a|0xjf{U9Y+YHt>@!Q#dVUi;(MLVX`v#s86S1dhe2K6QKvi1U`3 z<-T6KRtkynAAJ1$sAQ6dEl+B;UILGSR!GvDySts$2&HX9d?tSm-XdppZ9O}K;&ONa z64pO1Is1k;IgdadjkCD#7Er|0G?km31}K1MiPfRn1Cs2=D-|piQ7ppT1`ClEq(uI^ zo=*(m_dmXL-iR>MP7>F7Sn=rS9vVC6+#_-OY)D;2(@X_-c2ub{2RHLGEnnh^e$If@ zI-b_(r>d>ZUsm3;m@JijkDT>TIc`R_%5kf9GJJ6x2WYvd%zg{y>CeBae$n2xrS_YE$h}r33p@NY*33kFiSko%c6j)NeMfrTYi_?=;%^ zKfrCl%yxujfZXc|aW6nS62)^}9O_iV0$OSB#bxQOQ>(D}zFH`}AC-Lr&}bfBlJk7= zRMfWrb4<@O94c7PF^=~zz;_(v3X^O*a6!Whb#gTO1yRB#0gGRtk7FKr0QtnK!Lx*| z3u46KZx<%+09$@uT+W#P%qSoSa|f2>}EEDJT4X^5b9@XkEzuM$4nA43tu1(}B82DBiHE?QZ zxH{=n^J+B7EbOkhoK~le?ibYSS@)(uEgiw05JWDE2)B)`CG8*4|Aim zD{Rcs*?A09A@oYus+uWSAS~*OE7GCneN=a zp0QEAaf!hsU;1aq<-!FfG4EXED3+pVtKdEiMDhk7{-P;eaI~5Ko|k}sy%rxz-(G3X zVWOfmaSg{L>DQnf!I%Y%b?hBiO@3^XDuc9A{+sNH!p-%%j5(V*DEfoc5z73q?QQSF zoram7Q_K0nRHX?{02hAzgVi)#8lTHsk7u^*3({D%3r3r;qipM&Y^Dy}sWy8Kt^wYC zc3**pHr$jvTX=@|uf}6dZ~%PklyFhELs=HxuoO8GDZTS8F*gk6RwSf~gLt~Ky!}HM zIoZcxh}XL{8S7v$pg+lNd2K<8{NGn z<}S-B@J@B;T_Hp8 zAk3J>#l^%?$?DKd+|Eh07^X_}H91Q_rbPa0H9)rzB78I!H3KW%E|L~bh3J?+g zJA14|+|pnM8uAPm;8PbVw<9dCc11s4y1YFv?yuN-gPK+v;{1`kC;H^H6+16fE<(Ny zc{5Y}I&JWcle2TJnco7PdWyNn<)r|s7`N{lm>`fthSUA08NZl`LufIvr&M zJD;nF7SM7)ExY^OV|v!Hraf}(Fp+ylXiElCQOfBbHg=V`w@l;0@R3R1w1@>fGyi*Q zUrYL4U?46kmQR0jS_CAoI1=WI?M}su*)Lvlbvna4n@V;b{_*2U6AnL2F(;Twn7GAm z8i>`v&{K7C4G0(PG6CnVQOnYtTQ@+Wy z6su*N?S>o`Q9})-o$Av%!x^&nt*ao-50#HJHI3UG;?w;WF5@@$x{j7ThKg=pBH1jm zaoxWOMGqS@qEF2xxZ^(LSp!h_907e%7MZ46owZI~$=4+x9fJZUv z+V@!pQlpE(HsSU=Xl1|@Soo4N_qBxEoFH;s=-hI!Vyn>QqTZZcT z?*l$(BhU>Lw`r-^QAEt-pvCD{s-)D0XvT<~s9kSmvq4u$W0@wk?B2&Ve}uE&(SQU3 ziG);`dJbp{PI!xQ^|__S1YRMV(zhPn7`W0S8R?pDFt9npGy_@h<*Nuc4cO~Qlbf9! z*WU>D<=&X0iF#yJQn1X|8r`@vUO>ue{;7Pt3l6@5%1?p*?Z20|mP%|pZfgCr{Yn3Y zVM=>^e4_oNS#z?YJicf5po=ww%}7@Kg@iPfExOjf!(*t$VgSFIgq@ikYj?9jtCt~f zCk-tYG{EZbz_|C)5@jR?hKp`hxi5T zOgK)iBP5We=CfdP+B8UGkGLO;W0(g%9t0E|noCS#evPTUKOK6d!m$pACZBmY_?_ z$j@LKjvjFjSo+=GV|0EXWCW9i?ds*RF3F$ zi{kY3O!aQj{xx^7UWT2z;NjjrJ0hv7wvv*DwXihGlVCUBj_6CsKbljyISm84s=$Ix zI`Txqc1*6dOu6IQz*u7B<7|XzO`Y}6X7px1U&S~wJhU6Q7ZSKlw>*i~G z@nZrS{yc_c?Z`hp(VJ*LY-6$e)?j*D?WJ6|5gYNOt8YGViRIKg;EAWq3-&5#Dkk2w zF68TyO4Q^Mj8Lm3x!)ZV@-^m>*WxqTw_}=8sDq3|4gvG3r`)V=b?G>aI>q}7d(Kzm z!XGNz1uV22-*oS?A3Gd31s|zh!hgL<;KI60Jo-fljMk;2VQ!W!XWCk<0R}3*vMBxS6lk z6{g-JFm#i8bDTv{_u%8YG87=I0VFm-2CgR+KMB`vGeoylKht@}6p^(w(-&9>JKI3V z)w22LPZ%tN-a(IRR?7finN|5Ws=VT^EE6)u{gAIA@*32XIz>8#eRe{oGJz-so zr3Ew6w81TZME|Je#r#Xd73Mt|00pr(NV(gG*>xQ&cGU-b+PyJ(tC@m<@Qd;#4VWEV zTJ<2YicwcwV2o|sfrnpEoyQS@+lK*G`5**U|M__3WDt_?fYRCTmdPRFgN*X9Rm zW|*?AYeE}v3i#zF5N8?HDb&B6>VY;FZVOv2zk6Yhn8U`)8pfN_LjtVq%dP@1X&-i9 zBnwV1OS;vVHNnSxzVFXbfg{)@MN9|11k5=7i?N|7kUbn;Bh^lXyH^f;2P98uY=l}5 zvpssJ&4-Dy`?*y~sVE=q`U^Qg&-4e5O<^w3VP{-i};jWy*ARjM#&`M5SQMD+mWL>mGH!Tpx*o# zCK$spFl%Ku=S_@yrDr-VR9YCh-M@15<;%|^^s}1qU6}o4;-5O@Q^Qd_^gN35#Rj=PRtU@Z%6}0Cw z`}^!$08)ZWge$Rc8aR}jHqq~~{xxG6!t$$(O8pwUwD-P_`ItGIk4ZXeGb_b)GO*hA z2m}HU#?mgmNQHPLWR#F6&3aOFJP%E{)3s4AenHl4G}?TEBzhnId>GO@#oM2V!_~^t z&->)KG;E8jj*7`QYN*@+e-1#R;Nlm4JgG0c5{>Ghd6@Qgrc!F9#C$wgNB3ZYFqP^o zSvm~_kN>p#T?bCcJ7h8F#KWYE+Fv;!_%QSFiM7-*7*$M7ah3vFdiiE;-q?&kaT+`A z>hi)yPYEhPxn4&FZ5xz7wKff$Gy-|0iwz9Stc+yGctk*dUZ#(G^x4Gqa4MQ~ytOIX#paB?d0aW8q6f4|p!6v0l9X3cufGH|q>3KU}6Lsv>X7m^It*P}At$nL`B zf792u=%+^*7qJ1 zQyG%SS8;5-*VWRonf_7ag`Jl4s6pM3ejWqFd1(%gsz)ASKwHFXqgHuEz|otHtx-$r z*=4x5;SU|5+U3uMo?qcof!!k&_mm$4Co0$PQJ(=vypcdt>&Br~)Z8KgfNfK_c|~ia zq%siD80+#vTy{+GOUDR0p{f>VkW|^wKuUp=AUrS)QPHUELcb}Sz#S{K2I$EriTT1r z!a;FD8>j$7#fOd^6w9P##kQMss_N0vNgY*ZY_Cl^^;hu^NN;ekNVukN8kCKM4Yd_t zTnl&}smLZ0q|jSREcs|dIp7avtUewJW~V9Uh5bVQDO;rGMcptV4lN37y>6B`O|>1P ziorSrzouoxZ={DDO?}W_&lQdG(3U$mSXolh8ZbD`@u({#iKFfudJ!^XJta4g0yDh2 zrAxW7vEJU#3~tPqmJihlhO)~Z56g3?0%4u`$lc0s_B7m^U5O^yOF|7#Vh;BK;ehA7 zunQgZu61Z5B%yBJxTOJqBw%2YwU?s;Zj4a55 ztk&FdvkpZX&^4;eoOp$-MT!&dO&>b8{LCMi7xLCh!VH$NchTdGy}bpUXXgjyh^yPE zKl8M$;QS95i8k92f6LJA<{cF~yu36n6@@`>wcWOdClJhB@o=9d{~E&WR`387bhR!3 zEe?Z&X6rxrW9Bic+!@s+7>jmCCm-zz9fbHStQolDbFqYJ!a|x(4w_xo1Ov7u+LS{u zxqR^X!-J9(`FiH`=4v9Vlb;EBS2?0fr1Q6g1N?S*>jJzbXJ}@jb!rQl(GfMYM!BiJ z@yb}fWpxeP?#en2hwaf3hwoD#lh7bRx)y@C37`OnVduA1g3Bd9mrWcUOT4;FVf)M; ze6LfyUp!;>eq0NF>(Vl0nvo5BJF}5sTab}PMO}!f> z!-qgnymPS>eQ`T*&DKClJS@FhIxWTYOm6<^n(YX>onF5R-%QC z#k4-ho9Z&y9NHTBcs-LfEvKzWb?F#s4t%o|`BJFmMQHxUchRBo``U6wF&-CyaHJZO zRCLhhm2f=e;r+PS$z9wwF$6!y1#r!W)T<|awdG(?`j}V^UUREj@q!SMcJj3b$i{ZA zyTX6nc>e4?`(GkjFMeO|p(>JHHZw)`fXO`_8hjT#moV_W7^H(9q1k z>9YxM$*zIkUkXn0cLl@Uk2#^^qZ9smU@rr&Bh;zS;r8};k*glOXvcs4=ro|baWxs6 z4S!et6C(rJfuMEEUM6k0Ti!wK0pKAmM{rYpS*;IL3}z}e6fn&ha@Zp64ac?vX}V|8 z92+Cl(1v;3ssvlngDt&!WI6n7c4#0#(t^>aykw{q4JkG#>6#f*y&O$EfMGLyt_cYvur{ z;cB;n5V_rV2wO$Mwu*eQ3ZV~L6cw-^yHtUPRs`g+`XF(vr&2o}QT}&MtA1n);Fd6Q zn3lEkg)HZ^v@4)T+t?nq)6>!CX|vy4UW<{DJ?=VhAkpt{jbe6NKhh!mw+dYUy>(nqV#_n=u>>?nJ8)GLAU8Hd?p6Jq=U62Mk~Q(N zL+!WT9gZL_$n(tHq9v?^$mJTG9crUgpu6NzwZv>5PhSY$ z5Jz@<#REK7yUuyXp(;&;%p&l5I|EE2e2<zFKazIyd?w9%<4Ahrn*y z%(EWpnU}-yQ;tihj9qIpSs*Kj3_p()Is$gS-TO`5vMPo(C*q({^27o_&k1WG^8BjU zFA^{#IAq0OvnL9A7L9?zu{!bj%AvCD<0eyob2KYlY-Jniy$D9&-A+`k7=z{Td&C^!&o`*p*?=5!)4&Wj zbFn@;*9BIw*1D+|!_oKXD4z{Kq9}A<71OX;d=Zvmnz1~=0j%eq{pU~ zre^JA{qIB}>K<~rqAlge%DgnesoSkX?a&?nSr;ob;yZ-sPc#W%b9+JCl(z}f|QET7r*~nw-p)Hg6dP&a=m4^AWdT9RImMLZ|91eqf z;BTSZdG5#va!FJ<*9W}Cjm()2-J?9Fk>8`)2{SunC)Sjs%`lB2X2%(>su{Q~bQJ$s zJ{52P{`mZh*1@3K+S>j66qnFy8N>lcSmTQ?zk&A8fBs*9ZO5_HSLur!i#wQ=JI9rd zI3vMkxKud@(Qw-LFCfM-N`NeKKLPJ7eXorgO&eKTT|Z`~fs3|Z8&1P4K+oR6*JIgm zHFD7K&@u_w5{qA7Gg6jthD7@05XKHND!Kx?!0WLYr{TR-;~r+mBKRZ3y!U!)6JDb( z0Iw}iQ1n}jW#wys)Kzb{YbGrxe`1aHMC??~{iPkW1Ht>dhIQcTkpy-;p_yDl`1f8A2KRj<2&Tr~})b@5Grq})L;7OwpfmL?9IsJi$Pr!vWo#b3&Se5S| z@xTt!R4tPm>+5M^FGNl)NML{fJsP zJC_B_q0Cg&b6Eh=yUNd$lx70|20rZYO|Q??6BrbuK79?@x5}90ChoCCd7eT~#tROA zeJL(gk=?Syj3s>z!>M_#fA37|t9Y?o3+$_Nt$o|hz?^s%1i z`3)_FvEa+lG9?TnLplCM9zCX}f0uRQzHnauI zIm_RcU7V?Su6@=E9Gf)$BB}wj>;KMSJy?m!h2cV=-+Yx#7`psA2PF7DUpWY2s!a|& zs;!7eJ&wt(XgD^W!)DV*)I*8n@`WVt!(yKXJ$Xm{JZ3W@ne~JdBR?t^G6DqJZ0!HYHX%l6f8D;$)Z2n;MVszA9AiN3wU8qkwld|| zB}y{=pQ^8()xnMZj+`sH9@ht!wBdQRr#%qnJ`*LLFn=T{<`wOybl*OKCVvQI6NN=d?+m9U z3aZAG`|mHUobHS#UcOy4QjBs;KZ%dAm2T4-Lk2A0$8LQ;+Jnn&I4-BR-<}`~`T%6R z%o+IY4i+~23!;q#aqWi(et%aTkklO(QqKK@f>BS2&|{8|mjlC1LtKboh60pH?!GHS zekEe?@!J z{<3SES;>~sr?D~1I-8gBHyw-@@POrQy0)$AUp~DQ|A`5-kIX6Uyn@W(38=2^_@e# zlFSX=tc(iCH9I|91EBtcJkg_-py?{*>6}5>vODTDCwi%V_;L7a!ZF0`iXoRa-auQw zdg3cb>lT%(;jTj+?zf$sXt`71sD!KzrVS}e-@;DzqvcM$7zzSAk;8s6Zyubkk6{9*Y*e2nzcH%(`9b2 zb|97wMwQ2MKql_9W`&q~47`GS98_lrz}-6U*(TeB#y(4LhL%i_+#=tAh-73Lx_M)I zA2Or7=wx4;;x|7qG+8F(|AC~An;62O_EChk&#&x6PW!h3Hok&gl}C7?Q?p1RvY_IL z&^GiZWD0DNlPLLS5>aCT92irce>wF1wb^0l+J>nsPPRiQn+?0i25$~Lw!_Ts@Rhgf zo4ak-sx;kb7e4%DC+D%bP25*x8$|>XF`TMM?Tw;ZARwUU2-8$S+bzhD)oHot-PY8qppH}}V@Hv>SGQd~G;um(qfVmDEeq9%>!p)jT20YQ z$SbZbPRC_K1>jz{o4EcIo`n|Bx1Isk4-Pn6!eZpl+Er$>Coo4^R3bAcw1PBsb>h;X6w4n9RW$f7;xKH?c z>zS9t27$U^sPgz4nW0?1uMf*lJp=P$>ej7}Y07lW;&83(%y*#QEKdQ5wEz<>7IVOM z)Dz<-1#3v4;MFI9{eKAi@2IAl?hP2`RzXE6DhLP&A_!6hkzQ|9K)RFw2~C91i}YR; zkz1;?C|&6!Gy??bDAJ`HTIiuB(mM#gllc6;_5Ja#^=4&}ERvivGiPS+>$>*db2fMt z+>nlZ-Fnl1S`>>F|KOUEx#hQ~gX%Ij(_PaD);G$3TyE)S4PH^*>5m9{%e{gaX)j>KUsPAx{}560 zM)lX~v?)*mmmf|BE#;k^{>Ev#Bl@}JY-vqX@a{Z^*s?$Khz=aUN)@$azwX zghF5q7&L>16?R3G4eC=p-%c2CM79m{JouWJZ9Rr?k|8C2o<7Vt!o>v}B<$=;nig0D z^s#!y1(>B1z5eN*(~UdiJt|QQh`Q;%DxQ!l*#MBvM2sYXE<$Ox{&I~Um9$JgwWI!a zh5V4yi6uhk^l`&c7RTwv=e-Q3@Zc565PiD3S*nKL1;Of*!4?Sk3Qw8eC-Z<~fqA17 zzln1c%~vY~H6#_bWw)A+kDD$AA1qYw60ua9A^>7Oh(-&>g)9Pz>e)a4LrNYS{wG2R z=k9Gg{V(s=Z2oe)WDDQ4LF@iblrCx|OL6n<5h7%-Z-*u>Al)Ik%iy>ckLMVX{c{kv zn-+2qadiA=x2esd3ApTj4fE}X%@K~nJPV|2 zNsmhZOlV>^J!DA5Rqso7Re*PAQ%cghv1aVImB~okU)% zp=m~{B^nToXcbmxjpW2lV)S7H?iXJ@ZP5PH>;i6QjF*4I^4 zAEwIOpas8s-RSELAZaV~_AOF^ukCfE`GKItp)9PU#istzYITz>PVEaY{}=M4hH(24RkboBOqe=Yu<$h(`r-N){n4BP(-Q z$4fBr7v83gm8FVD8T-UC#f|FP?LSj_b!+t|K9hcG#P^c(QkZ*F+x(Mwb0)7{d0|4Z zMIeX#ChN_DcC&qmyQ%Gf#`#_`(4=ss!h1^dPT;$p}?HertV6m2$?ZGrwtA?}R|5GeTGSm~pD=z}b2G$`2kVxK3&YnI^@0(ZOJ8I?bSccjM)E;64lFGCAr zS9gA{Q%0FnmnZnMS&%*qWh5opQiKNW2E2^SHt45Iwmr;#f3`SDNf6pqq^OZ%qT zP&qQRsn0!rWBKIGEvw&mSl-+k5`#k#f{r^=36^?l7r;q0|L*!#2Li@Y+TKA8xK$$m zt0yzYk=ak1nAVQ`%W#Rr5}I%H6jtEKS`QYWjRL;u@3Z;xoQ0dDKUgy3Ua1X9ER15D z+83DtuBW&|u48}p-n^iCX=Gc_sGkl}0@3zUZYC^y9rJfjzEsU*q7HkCg?ashI)iuw zMRcJx?Gx^5>F23QKp;6{m(OeIFJh2w8LnZgms$8DMkIrWzPIOLAdp4bVG(27Qm44X zPK;Hzo0D`Ys9LawYPJF_+I&bZstk4h#yPS5o$#cIsiVwSR7<<~OojT8rY`gP@ky|7 zlEtQUevUdJF5?CJl9^gD${L&1ETH) zBL5i_9ig31)6eBIhHt*ankV8QNJeSeb|yoORGY{aV_k-IhiZo>?rf^PQof$r_21kw zFSooSesRwzkp@-o&X?|N2lhL2YCXua&vz*hZugo$&Xk+xNdk7XS$!@915Wym8xB=) zxh)yvQczZPspOjJm2w%CFjI=~J%bkCs(2P_giL7fkCJ2>)Kxt{ya zMlbp&!`8UrS4O1#T{aFS02(R45GHVE9J)8SSd{uSYIC2h=-M__qVEl?F|QWxTRQkQ zb`5OY({PHR7`*xB#i{fqj(sbz7O_b1M1!L%Q-6D!hdHA*y>Earf?s(8Z0F2nvbC04 zq7u4oi+aP>x&2T{(V)aUV;dQY9U$(Bw=QGNz#8hmWZ?L7v59ZyT&_K`*GA{Ce6>*Q z0w)oR40QEE1+?(#CD5Ax7%IftWbWFTAb>95kQ{krDMibjV<)0hp|=Tf9lr1a%S-|cJzV!n_`>y_7H5yApxEB3 z^=wBxw-#gDXYL}y5<+1Ujn)v``Gf|}r_9RyJmHqPVp>=yN&6ANK|&uyR49YdS_0FB zuy5naB3&eyXI>U^@TOV%XNKDzK^tD=JWE^d1YpIYzwK`+ahgXZ{$@sHCG5-(_l|!^GO7Z{eFkAK*2^!es)~czugLUH)3onb0d_%PL z%bX)(nA)pgb=4SdNq@IHmZSW;ZuZWF2c|C*5qT-4rZ6KmI%F_AZW!X|3<>aMI^?{qP7t+d^vmld4*hUyELzBhlg8`)p1pbf}8S&8AUcEar+DHF&331Sdt<;b~%(F%cX zv_U#Ib%diJSO~`tEg6&b9o#EfJ*$M606 zBu6@?6tC%XHo|dr@0dEr&gkdLIA**jUvqo8?2@ZwT5Ww}X|`Zh%>Xt$YSn5LRMt^Q zsr}9aqxPZ~6rGJiSk7NOp8bq^5?a9@dyQwa6T(NSR49l)`1YK^@4gGs*<`o~V+Hz@ z5gcN4L7GJpk>IaXST`bRb0l9mwu&5DU#GsMIuZc)@G$bN%|Z)$`dK~tdh=E;HxD#l zgGg*YlDALEhsCt{2DO6?2u}S#`7DJ9lK~kf-S*4OI?ycA(<>V?axC!!tV=M%ePdM7 zzQIKYppCZ{bmjUCt@sxulTIB{4Srzyc8+0=H%|ljF5CuEMzX8`iLcWcGW6%c)tpk`G@Y>PC(c8LuDwEbqDLhcCw$lREs>w{KjX<*24#nWKc3 zKG>Q?r263l$AhX2S6N}7xJyhwT_hH2)}C>+%=Cz9f&2^9_^R6_>_O=D(nOU{v#nL) zhnbR7Wmd8<$EvntP=#a3pjFib%ger00Afj9Al-u(hhDIM$e9%bp~3 z=|$$O@Jsk`)sh^I+Fy&xJIKx6w<|-FvlrCtCz2ZH5A)Ax-1~KWA^oFZH@DgpL-NaQ z8|F3x4|ebv^mUky=9?XxuBUbc|}RtH(tHy771Wi1N8Ev_aE|MTZwt z3(-Z47~y(3UAMr@Sxyg;=Ret)TGIBglw&E%-kEZ0`$B&#SC5|8@mGTi4xXY9&ohP$ zd~S#uXtV4>hrb@?#hNTd^sswyyb0pCx%nZv_aQ8A71lj16rYQf} zKby}m^`>ht?sKVPG^E7PS{4i#6ogRFnhSoPxV0d=0udH=v1o9?dwRc&4~Si^DZSlY zNws~O_e`U?)b9q#p0dUoCZA06Dmz_VP3WD}5DQ_&hSjjaNwPfwFG>GJJ+@ zLm;dimGpDPTW8fjD*LHK@$YTZ{bqVl6%p?JkCYHJGw!U>h!?H%MUo8_qz)+pm%O-8 zN;~QMP|tu2O@oW8Y0sH-FF$j>WnmpGqznfz#oG#YljKNYwaA3WN}fBB0an@JE0%E+ zVmOD8UoyQQGJ7}&0{q!c%qRj06AfPuP4nzOrA(x8k&${*JU_wbY17ouL#Pu1|I*H&1x$< z{&?yE|BBvdl8_-NPIkGB4Cg*=&8l_52iL-h>5z47>^RxPAlscj;x~iG{7m45PpZB`DsRWsrdj=`YV0Z)1z0$HM&08l6U>> zhZ^W#h@nx_OXrSEv}Uu`K5cUpzd_hL=0}tZH0#3sN@KKbCt|S}p{n6jQA3dz0LL~H zK5>!qwZg6K^ZBA_r(b-nWk+IO>;aq*>?)1A#p@P0TWR3P2xPz;eF48BK9xE-@nxYY zeHvica6^qoVi%_5tN;rM^sHji-%6C^VIFo2?8}eH6MCTPmTW6qZAVhCcLYd_qx**8 zf296*_wseSdkjQ@w^EU4!QIaSAp;ZUmu^8ZLay17b8^6`uiRg<%2V@SPM03>XyeU{ z1Mu6Oi_un{L4`UDPkG{J&L+c)MgPEaSj427BDvhufv2*qKylyveR`i85@@mOusa&J z&@k^;N@7UifofAMqq$w>;{;W7`E`q#Q!@8`#R%hZA)>N#>Evi`7sad zI6~3OeHsZ44<7A?D5Hhr%%g^Xbs4gJIcwDSs5!NU09KE;*i1Q3qU}IxATc=#iXY){ z69)_{5{@-{X-%)}&Ti7U)aldU!J(cHMT*2Swsv9=J?BUTV8%RTDHB}b(miT!p1W!f zc`Pqo6YW!*LD-~ZOBph>clrE?;t3^rIR5|JEtvN?+ml7{R^41a%_JIJ z+#P!)w^p9HPA0y?i(MLXyB%Y6=Re>??Nj+X^;*@k<)&{c+;}tZpBXrvxKuOfFyRDR z@QfVh2XE5(xGF_63-;4bfT_-F)exp6Kj2OZJTQUch-PA8$pxYg6Lwmv{J0vzQ6)OiC zp7nAAR1r~pAa7g+-jvg0oV#5{k>a^%!Q{M@1a62#F~4A>QYARfT<*K1webjhX0-RY z#6(w;KjeOdYch9ukrB{nzq2`0yDevA$iTl9I9={`yov^=U>mt<;5j~Qx-JQhJyZOD zd+J`YWq!jp5|5X`mn}d`#k7*|q<{`(GS!V|T!$BNb?^Phr;};`!dEYoxg8O-R8b*)gng$N zO|o)PJzo-2I502>I^->&6uk?lKKdlr2WikvgqvLYLh>kp!IQJ00eC|P`#q}ihmpn9@Hr{nr0?np26-gj)&3Tk7Vw+=&LjYw zz@^+GBTR~N1GL~so4;z~I!Qr&+h4**?Of`Uls|Uh*hoa0qqG1pbjK>jWz@hzxtDKKRey*J3(A%#XB+unniY-6?b!l~lU zvV%+J*i*FFP@9UXviXw)({{rg!$d0%MLx01k78h20LarnQHhVt{&|$F1@e<1$nJn9 zim0SJD<9EP(R7m%Z}rV;iIHIaF4eiq>qt1i1Lq2@AP6x5(Rd1^`T;uu$-XwS@FwVNFgFV;f^>6!c&0WUHv6WopB1RV{n2qjJI zndpvf(~O?XEy4Fob-)y}B&3$S(TwtTH>O$W<#y*&(=)57FLfvZ#^1jSH4eBoVABTb zVOF1Yjap^L`oDRY0nTk7RFPB9$3PV?*lkdSVcWlGsG7OZ74>k9#r_{Qs!vA^tJ_eT zNAB!KByZ>p&95mraeyNDBOm(d^7V+ex)pP?RvO(r>7_aPM2HzrNr#fr*;z8qdGN90 zlC$ZU0aTso9tNJec#!K@Fa!xG0ohe&JU&-Ev_Ia@4HkaN*6L76a8GZg*)>q=)0AO_s4 zgl&-^t-E5rC35b#*56sH$KRQ)nHGtbOv)QGbp;_WigE(k2q8=}7b0^WC(K_qna{h% z`!HzQq@z|OgD)EJbCARnW!{w{qEXIF6@L#@hbt&(Ycw%MxBTZ{#XoKKIz*aaKw5b_QcACcSmh>@-H!0&mvI|fRrwWm^+}m5_CM2E48z|uZ z4bKL;rLXO!U8%{+=Bun37+5?q44+d2int}c`i;tvVk4%2-yjFbrsR%V7uxAAYWRi3 z>TY%@x0=_&VC}2mFQJUiE8hNiapU|V)VMR`cVbGP1M?do$iZUXT&96Nc6IT*2ns$a zLhq8P-m}iUm5&L%7-yCiiD-m#5!U6NJM_cDtF92jp~CiYzS~c26flxC0cN`@TFHdY zb5|375_j7j*(gJg-_9Agey2hDP8~2SJOQfcPzg1x9Wy_ATpe6u$WNX91e%GOCek5u z-+QD_7lEZ_pTMuCmuG+H5pZROd2`*~2f`bbdnoOD-Adxm8B`o9Z@6n8D$s=4{XWCq zj;R`U%IunGi(Kl&2$uLrv@>QqB3iA;3e-8W7@%>ln@WgN=h#{}whUGV+%=X}{4hE3 zQoCWNn~efk?=A%eyWOn3qQdvPx7=DSTeq!o=Q$GJSQZWuof~!oBFt5x2-v!3+xwN> zV^J+)f4BIXPj*1pf7NP;5>n4xs&)5!WOT1YK*Qs(vFwsj71j=$Y#X+|TL*Fnwtq>@ z8?*gzn0#6`q*w(lQA$hYO zh^BK{8eQ2<()uw2U^_bh0t^t;2bWcNane~#ZVX_-&?Li)s!+oQuEX>Gu|OEdkAgUO zC&qdA_|s9PT4q;egf{)_+CY0Vw?K3SEULbL(O-3tbKb&Hf@!OsFp2@X(KD^q;nd#J zyK5~&ohe|JlxP31*{L@8I67408N;t~VJ*SJ{mB}v>WMoWF$E&jpdA^t)F>z_?(9P~ z!cHyjRhyb+y?kM@VW8jIj!+P)g~bm@PfwJW(LmK4pIGh^$prx}Ov8SZ1hOhJ_bZ-j zddW6asDdpj=iLhr{5=1;^Um?-9UUZZpo?1@q}uo8umWN2`l+I6Ala2dio9b_^VRM< zVNYNcb8;U%qT1#9s)FJA89k+4bIZrQgPO_;7mVdxF@9OAHddxJ5X7KLL5{oXTV0Y$ zRg3Rq#~D@_X07<&8A)c-^Q+;{7P)B@DuL624xj#M^_nPIdipA-g?&N!AGf6Ctx2Ls zS=Z$7^|5^$>SZ1E2E#=8%a3>;>4BzD;M^pDv3~Y8*IuG{>PC9MW`YtW!qTZKoJ|CT zGAfV1cS#o|9_lg+yuF4^^Ezyl++N_tY5T7^y|32P0HQPNq*DKfg&LWLM%b=yzp+CD zj>Wbd%ujiz6TpT=7Gz?9Hp<3>97aA^#|z$#d@d*4@8(of>M^sI*!O-LnP3K2mA|f9 z0m~85%fXITKlHkuGnV-#S7mnw(_zU_#Ol;@rUb%|gwQHppW_|?xxWMJ(~(OYxj}`r~$_OlNj>HXn9u z*}*EvBhX1pfaRt~DZWGQ+a-#?KGtgOWGYKFv0~V-og*Bq2t(MtU7f)OiOjRdY%#(z zdZrE}j;>Wm?LQcc-Tk=t$uHB?sN|sT3&~MFN2l?LUuYx^o!siMN`6DpOr3Z~{ zI9=T&{z$#a;E&w$2ewd#aEUrIM&(c0d_Z_4U|U)=rgz&)-iMuTy6}S5mm>>F;c{ax zEDUa{8xIG1`}jLUzL1UzNB-y5D&~o|981XJp(*_J#yOHbN`!;2xY$>J|@l5E&r!ko5L2k&^ zIQ}-8!{nCria@Sk@_fsy`)68pxw_G`C1_Llw#@Zo5>-Qu6?aR2J7G~aarP{@c%G*S zYqz|0RJ&z4Q$Mgum>VB&6y_RT_J&+`ExT1TX{UWsZ&qelHMLU6r_Q6s<GC3ry?Ae<>hBf`0j>w9)%L8^qQE4{h#Feg2A%Yo*(csP_F5HkCG zv7V&770iqc1dua<_mT!B5D?PZzbr##Nu04Feo)GVJ7+OTKWEmf8=IiEkE{Ljb{7aW zkeI(JOhggi2k>#v)tFGXXXf@A( zYiVSD;PcEGx!~ALVwV^VW2<4m{;oi~!?6sygv-56`h#t!^+K_@@-7Sk$C*!jwn#p- zyO)a}tu#>hB+;p?Yv++8Fv7~_n#1J$3ogd0cjNWz3M2#xXvAju+-Dr!yiD)2H`Y+6 z0HsiPs)P8A2NvzmoHMhf022bi>%Gg2$(JNsTk5S3ct9^=XD^?40lE1_W3+ZRP3sqy57Uu^w#-SU+AvI0=?Geb@@HFS=ox!xu|vaO0;^iR$An_n6*toWosWW~4tP$@v6 zhR4T$c~;BEj`y_1$E9={mii&GO32(*%JMgzX3by360cf0jK@Qymi!(vyegfBAr`&&X!>hH)VA&Sl@{A7eBn?Up& zSWzHo^sW-uJ|1#^iO>VLW)9=u1nUF3XfUhGK=V);C>gFURqFR5pPL5Vb8evcE*9fP zx$yD1YfYwxY7>V_1q_1V7eCy5k*1e$AG%{>7k8xu-*=}8^FUDXA|6_zTWsU8Upzd3 zKa>)R*J?4S8|Y7-t?#Jk!%!v;+TEKU$U^a=WO(;UcALy(p54{TWX=-*kF7^Zc73}? zJIeBgLzy!N_4QaJQmn8s)sXoyxRKPVxBf5##605dH{%AEtZR7Hr~f~yg}T=w;tiSW z;qUYC@;4~~gw(KgAG@;4t!9rX!mXopWsfumoSm)d?EL(O$1r)^ck_|sQQz#?km4pd z)RYqtL!s|UvGvkd6E&0eBzA*{Te`jIr|`R;IU*W$M`J$|E8`#>I5AdSiZ3)TiCt3C zzS*8hEcAX|^A5)%O7ln=TI#7`4SF=g_ZM{ZW1uzN)+FjKROTk};-twy)60X^6| z=T>^^W8)ux4gtTrsW zd2l9XCu!*sK$G0sRB)Qc{xAx{V~@0!5_YegHq7yB&daxj zA|eHGd(S~vzzrAU!5BNA5uP3 zd+RkZ6;5A@4fMNgzz;kZ;S0ERq97bv$}J1~f?L_Gu4;3}64Aw9KTiTDWu3AWxN*|s zV#G8u9Lvy{JGNZYV!RUhq-pLh^=;tlL@q#3Mn2AocZwuNBzan%RzTjb2b^Dfpho3k zrn9BY1`=UnukyHMbF0xcfoQHY%8Q|^7rzhHWx$!w-=={L^b#c(>|&OrpO6w__Q-d= zD}SwG`6}p%xV3IlWR9IFoRQA9kL88|;z?K}U6cF9Mc*GfVgm>-#BtDs-u{2LtU>w# ziPv<--;4W)oOa;TK+(Kw*6Jo|_daC_U9w`qN#FYUm0h-3*?Q+b<|gYKOS=-XOr3nA zGJJB`+v^iH)$c@C)R)$H$7yh=-%0Z4*fRezCCDRBCU73;BpT>42vCDGhrHpWlGt{} zcwP|`Ni{vnlV^eH_!k4;1+jjW@27x&{dM8uc6Pr_8ETIW$+azQ=t`2BQLjlh;#ns{ z>u_{Y%eX&6`I8(@MoHG8)gE@o19|-;a8H|t^kUr?SKKbRzji1FoXidGVbVHTZ;ar< zeYR%K*KB!bd3o`9mx#4yeM;N>$Q|#e4pKudzrTKuuA8*LnNI)YqK-5La*+At{s*A; z<*dZIv2~ZOl{VuvGMC2LdP9@Afs-x9`Z(L-FgJEn0T|@ItERvE>fhbFWvVwSJ#SXU z?$%>NHeZ5V1d!KynX6T|_@iRsISC<+Qeq+2rNb?w2)VK)$k7)sr>(rD4F+VHlGoiS zlegX3XZ+GDXCWv%?q<^rZ{#&f?Odc{5aqFXq^0tMzi#($$v{Gi6T6SGaxUBiDe0I% z^-`7oGLkc`gsKF}#2Gv1_8+D~$nh#K?qNK)cJg-+O}(osH-d##6bYoUs=pDQTG{h=cRuc)1{)Za3APL8m-iZLSf7$W`u_s5#-cfg z?*?ZX;}bv7)%b2cE??FLFXh z88-GRE;QdI@JoE3&=4^C@3^a2wLm@!N5p@ulcl8&)p0v=QmO0wx}-XPieKL?I>3vD z4av4tR9#@J+VhO-AHTMJpE_)keU&pTm@h;Xo`WC7tKai5R>Dj59ylf=L+t1$S9fem zzldIb1VjiRD^+?QNAin+6PV}3eFdd@x%SVVqR$8wwxoD5!XcE)q-Dqe2S zI?gQUVgUCsX?a>V#vEb}K)LMX*?af<=~~dZK-%Qy5uGr51GX|D~$bU94t2_eA zucH$$U7{yll-OMfOsQ2tbD*-7(pw=9`ouJPR5Pc9Jk>627_O&xT7`K`TR} zAL)#E#ERMO)U$?vK{R+dy;kC58LF;Xb@F6X-h5l^YEA!|js!l?4s%wIL@Q75?$7y& z&jg9ReGwd*^#XvIc^p)x*cg=dTcbUWK|@dR0)x)$_c0ET*oKp>Od`^- zC^{}nBo^%0%|GSTcg;=ophniKfixRM?r@w z5+r8j5jdnIoeX+pR{Kf?HM}A`xC`zh&&UyeSLl13QJS|dydQ_`Q?JVY$U$pEv3I;> zy3~&;;w>4PL^h)&9WI%5LqP=T6{kUJE`Np|IG!$pLsgo-S+GY=j+8ox?osE%yD)-K z43Fc%6o!f;Yxv(LsmQ7tH(#DLIp9MbO0peF(5uE?W=230inJ1U7?N)s5`*|Wzn4pt zI_~vy{^_7T=sBTMGeqpH$|!o0X%Q${Z7ggo>x z&|MPzn%=Yx?knOqHS^jJu^@U`-;$nR<3W{hUD&H0zY zR@+j1O@CByxi>wvx(PUfgd~*^hoppTZNE#O*Pn`I%9q)eCLQXn7N_NYYS=V!>CzGq zGyPs*bRWJq}7NPnlq{?ut0*2c@--q(k z)f7*w;_G(yxTQ73+!sEf18`Vg3`u4|Ck@xJn>kgB_nhU7#_$i>9yXA)`E;h=!a$>V z_BNUyFNjOM`*gG{yY&ASdXw^ViyrSsAYJJrQwLh`ka4}Ay2`rEp+m5n;oR&jtHTSu z!12gZxJVqQyTH%FOyj7>eftI_a1c&N5F7%Y%kX68BcA)QTk%vmq1)RDn`ZFnX2ep! z*6#+B)iqu%;Ph5v+nq-iYLEs&HKhSy%2h~}u)I(1DmUBthY_E--nqPUK6&P(u~dfp zVEUJLvQBTflNeByKf+fP!0lyG<{algKoO`RA_`i@;cJ%Y9MA4^#IFuIHX}#QMmK&3 zCMSqJkFXduN1ss+8BUySLAcAd6!+Q~VS(snfB&)-ys|EVtrxNJa;262Frj0{Zr5>UGM-zfg+JOs=J zHGRn^{JJH*{f*r=aHFnXT=8tgQcdlP&m}f>Zu9h_eSbIt_QxilB;?y4-G9LJf)Cjd z&5)dj@&}LJ-t@@|j=jEhx(}1mb+!Td1$70mq$ey&|11nTDTbzU&RWw)ekqelRyso) zaZTIj%-+Fq>WH-*)>ll|Oft@8Bzt6vIVT4nMx@B*(+HB1)t?kJ?<=6bwHzo`OM9zC zTt-TUlVXfw`e3--ib0yaNp>QIUT+YkbCrenb8IY~r>|I`_l3J=hH3jtj1gKx%Ynb^ z)qjp$f7rj$+Pb@M@Q7qo;_%l5fS;#2=pJ?*hV3406Jk28`H8=x7X663b&>cdKn_9j z^VnyLnVH^ywd%nuc6l37+n}`n_0;iN;A7ofCso`ghK38zZ5YzuaBZsbaVAr(RNY0f z{KS+wDLr0dVPcPfZdCnGPtl+q@v33M#=*ZiQJ=SM=rhQtw^~u zRP=knkwIDeyy~OHT--a%_CuBUq1nyZu0;W}bWJqo+oehc(!<8Qeu&GMco1mea1F!o*>5(PV-)vx_PEpZb02u z`p{#LwJoTQ^~xFZ%r|d3{wVJXF&z14Pv1YR=kkxHH*h6u235ko1@BM3t%o_=pV2O= z8O1rZNb$5qBoYP@ANCbP+*g@8PxD3|KG|*}al=YXj+rl#D@Ju2K;TSA!+8Qc1zowF z{qbAG$80{aOzrg_Chxtk($E=BpSd38$03#`M8{QzTU;}~a5u=`(U3`c_jppaB+sH{ zWb4S8o{8 z-72)d!3fFA7K9l3?EA5)66pNt7O%DXFI3b+TAzjVPmihByr+sEhVCAld7<|_gcKs~ zZ+m|);Lu(O>L0VqeYsD7qv+0tEtbS@XD5j%-i&DZ+LgrwdNt}rrhTl>Ga$pOa!~v5 z6Z-beL5Unl6sP-A$osW$j@}*6o=>nFv?+RMWF=2W^mZoR8Jn+Fq|gRmTzCX^tt zxL$8W8WooFsaMgwBjV$Lb#`r$SHFcB-*UsqN^3Z>7U4N`16pVFixfsFCS!k!^0P>! zd97yc;`Q*dRsQ4h;lh!1{%&*z`*PgelL*!GTT`eG4WPUt3)f!9fb$PKOGa|#WgME8 z>HcWPIA3}A6e85b_iO~`1=di^eZrooSHKv|gtFbOIL9YKMzQ&8QuP1j0yKQ6uI#-8 z=BrX!!ThzQ*ala`(haSGk&@5*Xz?w99v0B9(39`HFwneb-B?J27<2h>hjQH*)mCc! zQCKN@&Hqsh1ihEPCH?y2z@f39x!mE0)A9lrpX#vUYKLG!g@))Vm9a=zVre&eJLbOL zaL-So7hFQ@e7=XaDlcdaYmuNj#H%E3X-R#~ILutfZO6Wa{cz!TvDc94^f1w6toJ(^@}`w+ezGG$BS(dvSd@Th82tFK6|H_Gg~RpFxA z5bPtcC??UiykS=e)?=qH|9V&#axkLe0go;zL0v6j*u%@vs{|wryw6KD`Nx_;diT6M zkF4BzQcgnoUGki1MxeSDoS}H@{jd%&wsw&CF>mR5cIQ_e++!!@{DRA^W1V`tN0Y;! zrexrgnqWDvSm&=;Bk6O|rI6t&mBhDPOAIwnddItKpe3&sSL0cyJ0#5^`ly-_fiW4> zDfME}I((d0XpZFdvRY>kWxHr25fYmOvP!^D-Wfy^<&z z>eOu9B+yNJGxEB;rZ=dj=JNTI9XjLsszXfcC+iM_vmepc>fRD!)jO%rozc%vzS?ET zS6u6G9$)s&==NQ0D=(R%LG7+q($F^ZKCdL1?a}92_yyKK^P(zxJ8!o?t)`8MjX25Z zn@NhQs1Nxq&!i`M8EOp`mOAw0*8odQOB)e6cj%bjM;)Vjjh&5F;CcYgFx*)?>AoNj5!L+L>J$-T5k;2onK6T)=;XF^SM{M z&&$)jFyF<~O(Eg2>_m|Ms+xE%H#-~6^!N2@FGG7?d&bA!0W2Q>lDa7H_h#izpKZXG z9&-*4FF!9H^St>aH{)@brN!`g0f%+&*K%R2=w5})ZVODYXRlC*;8arWcvcN;x~p$f z(DVn{NRlxOMR{wz#m&bq#}m$JzyThPg?BwS9{^JpOZ3UAiZyRSjfoju*yeN>4oj6E zSSCVzpfMW^_hw0{WEkT#&V;V|c0Uf!quF0FuYb`6^ns0iI+=4^ zgzHWf)V}0e;fAz)Psm;>p%O&t7#zY(Yx{4NXH7nfKrd z_w8Q1SGu-MvBq)lg70w$>CQ%>*qKBi0G_nn`?JT-@6(}y0Ie#h5=$M{D4Jgz1}-ZRCt~Xw4Vc3vG$KUsetYoPqvmT zc%=w!9j}7JcYb5&U)c&1&l$=%UkH+V&>Xd)Mv?+X z^@PxL6Yfn8LXi`1Fy~ygo<@bmN6LUtD16EKFvTdWyM+!N$Er0naX977`Jdp{!wUPx zK&4*@3F&!4HR)g!-rAKbLso#2iUTG-^4pmP`5c5l3YK@(JI-#yvMVdEG*Of@qFI4iY zG_Qw1U@>v=Z5}R7+rtaJP-2L3H^0KdyNQw$#++xOtoQmQ3%>dcvdnUh^h=b)?0Ni> z&*OM~2sRt5x+x^;O!z|{FC+x4_R3U-wWQ)fAcbP^rAo!f$+tr)K{hdfeI(+udXb{PBA z5j<`FD@{igL7MeCe~sacminM^`+6)`qGD-_wNkA1PP~y^yiC&J^ae&TsriV{h7IQV z#cA@#eqG(1qMw9_Y09(}b?F}6OkBmJVK5|bAqnPuzH*D|nS=rsD>3FhEXU5zpD~15 zLo>;(nD?JmG6W&%JBmel4$Ab0i!3d2@vbQz0dkHsETn5=_&4Wt)X=ge^r&kj7TurG~M%;-|q1_Q^fCDC9{!QD#=J2d|<4OTWrcTY=W z@b%1QPmcAPP>#3&_^@2&2U$jRx(ll~W0I|`&3Hx?FX2#S^w*{)lM}xwM&SX7px5-? zG8^_4tG9gp)(xsi#!JVe)275soF4>4Fs3vWV+Ac|D!)6h|& zO#WE!Ek5cn1CZV+wd~ioP~b!ra9i8YA+G+{xO@7d>?8a%$||L;oLa)gNn>jAx3B^J zs)-(imV~?L2SED==J|Wc(>+XbukCirWIf=ALWT$@g~t1L<4;Tf@y$M;mU%`J19D{(la?=1ZP$7H}L&!Txf) zTGOORN`D{z-gl!=_+Br?&CwpQ=h@f45}M$c zI-d+FK^9}hloZZUaxi2{BX73H)4myt#9Z|~CbXgW9g`uq*Bf4Cal|(XABiU=9!3_S@gM8nd4oeW}N00mZOoOti2RYCgT&4a2LIqD=$M z16}Al{oM2sO=~|;WL`z)4$YaYI_@#;t@)B?6OhP(m9f{&&uO~QhVNYXgIna@HTB)d zk@BW<*;gnObi~HIcdZY+5SgAFRXo>mH}a;!Vq>k#Pt!Fbx3}nf`zN}?{88#Lbne6a zktKuN2~~Yr(A%#+%9R;lW9sQ5xlnxW>WRxoha3IYFgvj>$p#i*%GJmtX|OR502^di zHfl9wUnBCm()(uP_s@y(x)4buQbp^L*CbLbW~6n!S~$pv2p$*(?*b-0Xs~kzf|{gnxmBBAi0?jnqX{_xmV_M zg>SF?3Pjch`=&#ImeBFjPo*1QU+C#}--OvoH6A;wR1JAn>uOWsZh65k$UHhdS{y0R z=Xd4!rJ@9A?7=xQ2MR!zF1_puYA{DCHC1ffuc-godid+>hxq|Zqv1&dN9q0FT_r(2 z!S{pcAGQH?Mc2qhiN|!vHzQN)=4-=ALET}k(xz&I1|jPb$h)R%gF@6a>(k!iaRkPt zAAx_EbMozfif!+L21jNO*Cl>Uw46`Q-+SM5)`(aosxops67jL3q9~TOu4K`oXItHN zNndSS1gC6x>osu| z0oKWSOjN9u(?wE^2*oovc@^#b^wpEQz(7YU3eT{f-AYLEx+-n!gOIIr} zK3HAGhdjUCpJ$)j61JAJ`XHl1MtY67d z&4)A+f-E5^-3Um-E{aGjjf5=W2h!cmqDTuXAl|9&ra!=PNLG9M1V9E>@ z7HZB-#F(ty-gyq#x{GqG)OmKEe)V-f)K%e;!C1_VJKZ_;gaM@`FK zNW;7fDYgC|Bb?HFZ;L2hDsx=bO^OLQN!_d<#&CsfaT3?p6Ej2>-mB*(>cO#KDROEW z8@5TO4n@p74VetUd5&+m*Q+8-%7<0y9irEhp-H{S`HzphxEZ7Pi&dKFf$tG5%)>40 zn4L@^)LU`$A(68<08k10U#wn+$w8B3Lm7upc7_tKL!Zm_6)E5Lm`rur?!tBDS#0u5 z^S@UvAgcEhALRPbv=?%B7k@IQMUOf-IgobS(p53dW_FZ9hUmRD;uPU=G(rp&fXN}+ zK^M&U;5Y8lVZLAt`d)kVH>7P^o+If8IxXcno5RDva`_?2E${EbF1#Q)vm`IcDzMu# zwarI0J99NpW4n9JxZ?p2p>BsreqAcc zeJY6c^29hat7%8GLO9Hxs(8iXw$3`SRUzRcOEnaB%7;pqkARnxdN8jf&hR5GG;jiz zcK)i^k$v9pH4s#*z=>C0(IiJc|&Y-;;He(@S=TFU0myW}{imYT5YK7PHW zyrB-;SScT!AzB#@YMta$@W}HmdOg87{$Qfe+vh%%OOkhSoN6jBpz=Es)~-gfxu{L~ zK;1G{Dn1`d+uYCnDzT=fWSTvmDPu93vtzXRFm9@RbkZ#P_CM~IpE0{b(X>$1@tKp1 z2IipQ=$WL%`OJq8{leO|rxg}Jt=50v2dKhd;&WDCK462b6xi1}jf48y?e~)>}p$OI@k$8DSWbkv; zHC#lUUb>!9r;rg4CDLzcCL7)Plo%Vu9s;EX`gt&ASTeMrj8mf+$^OynQLQ!YaJN`C z^Ox&7hdg&`(2B!rh>1Bft_UU(jDF=mZbJseTX^xz(=KZmXM|H=i#7Y-6Eg zgZz9wh3n5!U@YJMD5$hxlpxD}KHKS1ZT#~0H%h~E=jAM1TSwU%z*p^=SNFiTR3JVV zz1*@tVD{|O1K#MQD6p?~m8GKL9(LB$%-Ab*Un|LwMfx*rs>q>OGA{{f5ya}HEMPUyRROKtxfGUTBbdN zKdMT5CVxH1(Cw$)AX^Aa3W+^*5ueJw~{HRzuy&Dpm{{0UgZAi;sLmbdA=n{uj|PRakUH zgtw>!a>8q)kt9jwh0@|Y$xwp>rtghYtNzUfKWBcac(tuGiZ%Jcqooo4$e%hm*r~Ek zfl)qNBD2fQfA$@O3VYM*n(i<@u+kjANTSvc==Qh-8Tm&`#U!?$xqrvmSZSgu@0p9X zLXhEpB4Qh=XZOzc&#Pi&1s#Qu=x<u5 ztNPyQlZfPq$W!*Nle(*G8?{WCUW)N7(f$0ecf?A*qGn;&0#qg)^Yn~zIoY9Zt{z5e zOs&q}_(T45nu?4C{8P*w3f}f+?|$v2k;}O%=Bl8K#hS**Di0{8sGV8IF$=PBqRmF^ zMl3SZYU&c)ES?!@#d=AXU_`J0U<<9V9IvyfTI$2zJqq2Z)|W{ z@OybfI^|middhW?SCB__X_j0N4KSA{&U)vDy;h7YwyWd&AUWa-A8<39bTsg z!fMak50c7H#@`Jkx64VCTj(iCyF|Yd+Blv+vMT>9!VD^jQ$6bfHUFsgGh$a6MXD_f z6VsDN?=HoWY0UjJAUNHM5%~?WZqd@-Lh4V_ef z>H;@fvz_fgQrxjyX~u^Jmg=RTusKC~HK`HK6(JOUPHHJ+&B%E9DDnVxrx%Ux86B-H zTTmYb(kr&d9+>?jLMkkmL5TU!eP z$szK>srs+o{O-CXtb=U`9eF;vUb;e;wYcDY=V^o{>S=-RBeuhgoLeIEK|l((CV~PV z{zQ}*t6_M6m=bw%NUaI|?c+9%ryhfsr;5+n@iPHQpH|;HIKw2#g4&iGW?pKQhp-wp zS6JIV9MX-%h)8k$v@s14D!P~R}84sZ-ui0 zgW0Tjl8)yvHG0QkfF;p~h+ax6p2RepT76QCfmhF+S zEH>PqLFfbKhgOCibBog7NWzbWPq-yfl)2+8pIGua325$l^EL2$l~@MSD=aRaMQM-* zIlg=A^xI#$+YiH7?aN@`>geWVQ|Nj}O)9`}wkhgc0kZHN&QruJte?|x^n|Gwfu7@} z-!HBZ3*5r`6brr=t#gA7nOIy32Hr8G+|zRU6ZzaB9^L+1t?Gb@LdXO{o9&?s6LiSNv=_4V|o ztQjBo{nsi58tUl;C3r^!O=hVxM2F^D=!(!GG_Kcb+KBztk8WaqPK)GtMs$)!GJ5w>C*?!hgDD;Ljidbd{){=! zU51X=*+woeuLo9E&VWdz(?xEAomSw)QUX>+Ip#&d@$4B-X|c@F?sh`!BMDo^ahtZm zpd*rLoT0n7ch-#GgoJQ`sQbSb0NcjD$~ji@P+Yf{a8YcK1`9y0y>(8Ml$2Vi!QaIk zpJ5g5w^Olf6&0rgLa}4fOZhqdi+gozAhOlCKY#v>C=fT!q=DGEpoNAbhG%WdHCQ*v zw`mSV&?|r&i4Y9R0*%>r4XCzsjPsDKcoN%-6THqA-rcmP2l@+lRHG&ulBZPIXI)@( z&Izub9zYSt^&b+u@9T2J<;$+sc5TK$ARN=&^REI)3WZ zJ+@*wn-{oNAQB0yI}%iz|CRQ0I;X(J)!o~8O?9n_|4 zc3Gm?A7exD^qRB%rL_UeE^%(+5#mDi|9tWWx1PAlI;K=%WyoQxq1~WHNmxYvR&hQ9 zdTnc9g?X657@zm~o5v{6?tC#^`^wc6D17s*+ZccMMSp!P`Yd5`TK$_>=Ajvv2|!3X zLX*>Gs&GfJ)Ap0W&S{mD5PILlVwpF^*C@iml0$G*LRjckFIVmJTG#k<(!#t~8$3XJ>_k3*bAt)&+^OK{qsLZswvLvfW0#o5HPUgh@ zN{LiAjA>g{U>%-a7;KH6pZjsuOK*)|F_PVikr>z@+ol)rQX+HFH>`Te%PT74Yb@oy z@3LZ5RJ4R`(zTNjqT@E$Y#LZar#R!DZOUEWcRs_8ayY{5B>63u&`k8UuReW24pQ`4 z^9#nVX!X^Pu#f)KzEqE}r87Bwd9PONbx`hR_M|44D!q9zXEMQdY|Bq=^t!0 zCS_?D3fPLzQ#2zgbAy>fu2kJsS}2;ibHZyt2vv1+~P-5%=Bw2ptB6zEZ%(YV#EDRs6tkfDX z+i0f9g{OI#h0iI5^u!ZGex1qjr}QEorKp~Cu-MHIv;<;?9x@m30`&hy3g z^S&>tRwsyanb**`vTs{(=e@0L?St>VDS}Koq*yfP5 zVhp@1KfLJIUXFFH&v}nJ_B(~Gr^{_oQeRS7%gS9kY59sm94OzDvKOSF1zQ5dAmVGvonYfi zPFC)9TgS>k_?INSB5*D4>sMz9Y&9lGyb%BA_$iM2xa>T?0i%|>e+G19 z6rnmx$+cSg`f;qx!CX5BIZszw%$y6XQl0UupBFveK z+x{&2mI^yt&Z*CTK?N5$&o=w%>CSgN@u!v3mo_=AxLH(SNaz1%e{4M{iy_$e7Toke zVa2Md2s3JyEL)eP%y^iw9E6a1%%DgsX5bwSSah?)1qptyXsQvCGKbHSbwWEf%mwS{ zo>9Lep|+ZPq%^-Iv%AqsFRIXp&iCl#v78P+&jbkrMGr#rQX_9Jlp9Qw!$ZMsjR$TK z*W8wf1&8wX1=~vfDZVn^9}#VmdnODBUAD4rroBtVx6LS(6#Dhyk`CPFV}!z#<7Cp;Gitfqy8xY(hUn$y;Ovi9s_ z6Ytbs>ZpG5IqhCbTj32LT^8%e? zJR2%!j)2_BA7;7h_<$2-^CDIe-?onUd62}cs-iF*K*RN@oaHYpjLSo;3_S0~moYk&Uu_eoa)F5a8Y=G%P#q_Xcy1`*)>`Jtc(LE(q z<|5tN?flsDr?}YDAbVcA(G~NhbT(pRI$Xm^!6_p7a2IHgk1w>x(jSqA7lMyllgFz; z)mEdYNNO@WWxdRWcew!1-D4F9!{e<6aey_UC-%>xa`|#Mr1UQ71x^h8ceKDulr79$ zRbSG|=|4Qc{mTUMP?usWf!?XYVv??n>LGUTD)p6U^?B2TjaeD9EIj9CeXI)<-RF#3 zs%Aa+n)uiy_#babPm;Ed91tnu9|$Tr8FC0z36^OKd4-8S20gNq#;`$@ANlZ_LLpzep5xo8V`s8#uwFH1~^ zPCe86Rs&y1zZuiudj51tP_j~ySw)kChlPDgX&-Out^OY9`}(6ey5|RI@%V#>0_4;F zw|40IlUO%HkSzK=^L<_FrJxCk?hlGeyG_xRKBBTeXrje3i>Pr&JJ&s)iLCj-_MmRb z_ySLRycaN-97?jc%7C?&<}r#6MeOk-GWgHzk95kjiBmDktsXt6UVP- zaWe^n8V6o0DrY97Y;4vs#==U<{mk#yXFBuKQ$l_iqt2(>RddqrJ>sVG%Vnxnmwzf8m4tOpN4dL_ znMu%XB09~};i))3Q5qNp{MnI3pR2+PQx5=}eL|5ClOVD;b9rTVVpQS}8XA-7c@HW2 zJTT`;RAvDOt|~;He3a)`t6o64>F28>E_)!El?q0g;VxgwT48>Ou?i}|D{K1bQxL*> zC6*di78`($LD!=?9*i4~o)MxT=rkmVrcu=^aQTNFI{ z&<(nFn%3lYzTE=fz(t+A4t)$w0Zu5v2z3Q;PpRaOTVvd-r{N8`8*;#;H^N6H8zv(& zT0CHP{j`gGkV!8`BM0flq^Gq0t0?!i{jswusBlC}4UmjIImE1G>%Ic73@=Eu4h@$h;H5;r>xhrylzYT70KG}T^Kf$NX4@{Bt zQCvr{1?^x0|04(FYTp5CY`F%R313mz>S_tdSXiF}VI6$zZM=dGX{Mx@_q4CXQN9hb zcF{MrPe01UAZoPVD=2Nx5m2NV=&#ufNXf#9VobkZ+ zNGf>qV%xyEPCS^iTiMXUZY+J{ca{K$^n&9pX{t!pC6WP@Ite!>a=oqvw=R7LK!}hO z>>I&N)+m+4Q?uJ8A ziBs$LSj76mHg=(Wox;nMZz&Sk5_T6laWCI$%^RErT#Ws8co=a%`X=T3;ePfI1Dqw~ zbTh9X{)PE*@PxbuP3P-!eothC6&20ia0L`OyyP+V^4>9v7PN2+c7_n^mfV}7hrObE zY)sZG+YkWIjia4_ZV(ehC6Y15S=MB5*QWH{7dDrQ(g{;K6@af12t^|U8^Af zUTNp+$vQ@gx5vwk=5-}Zui#$B#o>47)6(H*v#0|pQuzAOZ(nZ`P~p=5MTHTV4rFQO z)9^NY*Kjdu0fF3{E;S-rI1`*Y$a6a2D*IA_pKmu#TW}$mLzV9L{Z|)C}mj}^U?svYw#mT2S_tH zAJ;WV2*dl~2hU$ha`8V@*AF`5cIhhX4XwUYoC*IuFdF1q+4w2L=1jbXT4{(1T%}+R zA8wDuWc=hQmU-`Q%q!4cR`dAHO0HxB0%D8Df@aguSNEEv8BX7CV7F zYG%5K9Qr#9*~TcQE6%*)57&Wevx7SS8WQzgHN$0-tW`3ka;qe#V9RMZ-K_v{%feS% zF)8SGuZW19MM_*{!nucb8ev3$N7MWuJKd(icL0P}ns)N?PUPaSr+SOjT1Tbz z$D6sR9~(vkLs;RU>&D_WVjabW;-xlJxV;?(A0U^S#E#No2{`dc)iG{DVsg#j%nvmA z2hy{_-(w#xmA`e(uk|=(iuA1c&C1;mD;?G!Ma(y|UJI7F@gkowV0d7IiJ}M(?i$#x zuu|BTUvtHz`55=2eD)+Imd)(QxvgJMqCgjW)WB_z5U)AO&We?CMCJBWel8WL&5>U4 z=T8q}c(7AcD}3XtT6|uST@TAKYhiXSG7}}$me_r6JpI-$s}_ztguQ7st-+&H^p}h) zg&V7Ki%|7Fn$I}2esu+*kAFzHZ{TGc>=$3qSDN4$#KUfc-@;BSoM#T(;x^LuYLQjY zuJnFm*pb=0Azbb`eg?Vw6jxT_qF+}D=#u)&-50B#&qq?|BQ0Ml?wYSTb{u~p^VIMx!QuWl~w|e{zi zLmr<84O6(2s-W5R#@`21e$^E9B0N)5ZKqPTdZMhWgVRy@sH-fAYbY}v%+hh-eS8-0 z`mP-ux4)J`I?voc+jr?z&PE9W+t*k^)NB?8@*h6UB4&9IO=7dnKuQ!65(P>wONKRt z;}=`t*;~{gpz=c`za=&xW+z(Z1h0icO`7oVYhfCE+OGF@b5PM zOUUSRzqA2Noe?UH1%7d=`_G&1O&VpfMevJK^VlP9%@ox+?#&0$PqZEt93J=S3@V+d z&Z_+Sei;HGpL<1!P(mPFAoV_o^}0($%uiAL`pZdJn{Uy-FXS{HZ-18^AuEqyD;DjToSUlxL`!~YW7mhyFa*s zlau9YFDfhMeSkYp{+BMc84|0jM3o<#`Ri(JJGd|fZqn@6-(oQM6cb0dOz`wE-N6g; zLnIJm2_$7qnm`HseGv?<_x}foK=jq;iN`on6mY;m{Eo42}%)J1*gtQPwM~82lGatr&ItN(&9&=E0utz!>lNP1 z`7~o(ur?j>Kyi_%lcemtxBKW1&t~49c3T|sG|W!TMNY#{^|60m1-5da;OA#)-Gy~^ z;bEqQVb98XyT*=D&NLiK4@49OnNr&+nJ~%8u7&X=)x-yH*OHxaZcz<~wn_3awa2ku z`yNLJ)ARb(5+g;K)Kj*u^oHBBJ|C*q^zr{ffsFa8=lgz zS&pLPal)^m&e8(hB+{b$>c8u>4lAIZq=X;YfoQQ2p_G9??cLFl!}$#5x;o`CMa>#| zDFZ`khoXS5|&XpWFCzq;T+#qQQDS1>(R>*8q$k>?V=d^dK*c!^KqY%F7TfXOB?g_0+`yuIG|1Idvub#BnK#ZUr z1|#8dF5V$RLhXX_8)!VV(Me7=ZCxABk!breV-v8K1Df(4dd2T86I3h7o69>=pvV3XAHOQdhwkR7i)73ePSM%@>HKp>bX(rQ zSZi1DC5RHLuN@(x>LSiZceZw0rmXg;{SpE3{fQo?snbr+i&zUbash*Q z<8N8P@^m(+PIvYsa1?Kw=djIj0geSRQZRpTzTa1j@CYa#tHyx8crwTXrVkk^NF*za zr>5DvNw_|-%^qfdWPen?o8Mwg-n&Z&*iXUrEyCr2p#si$W8YNw`0C0$SIIsSYmu3&xdmIYy=KML_Tr#(NN=m^a6@zxT z-yZU;=DcZYuL}HcqMZ%g>uXN>!v!39G3G@202(EzA0KQLm6Q$YUd4UX7W9=o%R1U! z*2xrbzgT_AT>}0!CuzBrzxyJ*xIqX(vLBZS1Fzram1u=YbY5{JEFpWxQQH~#<8a>< zFFrtg=-PT!;zb9;xLYu1cZyN-HJL_7UTc{9UcGM=EjQ{12zZ*@{=5DVCt!8-_G#Y+ zf6Al_p`Pa|4(2aASu4g@cV<-s5YC8Du7IWL2HgFTfxTx~I#gg|2exxCY*wgOEkU?d zFd|2Ql}WqAovpEoe+{bz$IZE%dcYWMhuPLQZJV+LVdihxVhxrzqSr?0W>DM&_t}Q6 z{piq^W_Hj6XQJ(>arfb~Ln_*~J=+cwUREnCuPcwTi5W0@TB6Q&Ek0re{2T6^lHAVAn^DB&ReQ>?W$Zv$Gny|jPZAzd+sz>J`} zq=b00NRCZ62o#rG^m>_574DEMPFA8##_?I0%7Ng|RZ?7!Q4yL9pEB zae^}#GIhnSAflhAOYU%RajD)RQ2PSHo(`%59X!^>S&N#N$!Ak9Rq0Go!C4F$dG zYdtm}WFMcM7T!}|ras2W@rjT8{&WSRdlis*iTh@_REh#A%6s*5 z5Cb_D>v5$KP6g;HzyHv0Z}K?fm|jIPjUv}zCjEgR!6GNrz3MtW82FoJMqUaKs8)lg0Aq^ zZsNJQ6>--pkTy;I%6e->R7mB6uExb!|t>&Y-QXjt>{*y}sPw`GuVUA_*P zVFWZiqdod?dP8#ZY{Mpc)%EZSQHvg>$evE4e229*#NR{Uj?AyD^5s(mB0_(E@?dr- zVGDj!7y?NkWY1Oi4p~>jEA~$Zcm*(7;UXkpd1j@$26|$=T+uR3L5wUQZ)CY{b8#;e zVN5w*uvL}jcVElh3GpdBGL}7@$2VrMa--Yb9F_h`2^RJ9!|p%220V2H1{DhD#Py2BqYLHP4Q#L`x!3zr!E*&ntk|zw!ja- z{e)f`9s(v?q;DvATrLI|^OM9iojM<2`l7~Njfc8K+I=Mv6>RB6Yi#+v@vt>mZntbS zqTl2^=RVJR?~v4lhS0*8v0^O zjw`~8){K36dN4ZbOX5E}883O%f06 zx(#zbkd0tb)r!0B2vo(G^d~Hn?yl9-j+6?6m9xw|69A<7UCTDDnCsqXw+MK2*?Y9#~V@Jge>12mA zpv7F;0I2^y>Cz%`qas4~7*!8UI8e_rL7EE*oYFJW=xQt+b^vF&o-H+0T(1o{?S%Q0 zE(5atmzjM}*;#A3SzB_F_f%{9bmSh&%}N4>0^B>NKy0pYW|FE7{>(|blVDhHIBjgN zzIo8{I}5hlf7h=|qj<$Ogk{{XdIr333mUB7pQDSOdC&?~XUyH6%JcSV_jo zl7VLn$oBf~wBSpS?cVs>m9FFdexP~)rIwNai#Wv>zjS++F0N&j2ojV*j;MIb9#LIF zPZ1D~B*bb9{tbQH6(N67!l^|5VoZrF123Z%pbnsTUBEO>*OWh>Ai@9w-2)`pe2X-i zyI9N;aBT1l5ey&`vi?Pjc6)vx(}3fdphlGZ^CbQTqrl&32_(O9jK{3%b z&83az5dJ%aa1XU+XH1;u(}ZE--hS;1hg8EgKXnb#gzPX89zGOMC4sd62oyV@#R`sz zb41j!W(;h~2{>VLIYX^icS>GO8r#Q_kl(w9n62;~!+Y@>CXHXFNHSQq!rP@oIhDMSDqy#5;7^Zb;74q-Dh~~yM(zyQ* z2%&3|;6TcV>$enmh)Mc4dmUjHVr@T z$={h_S`H?96$f`cN2TE3o+VhiR!B^w_?FeISG(LOFVfrR`i3aaq_K0`hh`+H?MoM%E7#!b zcDBZr!a-K{k42nN9@axN^#Hx*CHdqqPu*(Ag=8LlE^$4*yo{tBW2y>CcW`i4k(p%j z=V>%*D(v%|TBQ^{g;Tx9u1e{z)Q+06e^>~sG2j6F;%&w|FAc5SV;<}a^|6dH`ra;t z+pFxX#53Ebm8kl?o<5Y?C-I*LoA|jw51L+h4(fP^$D-j51bCgdnzrVGK!ajNMokUb zMyb3hfnXK6FY_@Gi%<8S7)gEqOZtSe9dDUbP6JN#PJ2A3dV!`k3FkjqWuD_S5l@ml zUrEi%%g;fC8@Ku?;9{4Kj+hVJ4?d+`|8m%#hGrr0s1K&eZb6(#HJ!gAyH{H7$GYVN z6GRBx@v5z2_2ZpPzsN5{6~Ck6yx0*Tm3#Ypcl3OHESy@(2Nvbh+viZd=;)^#j!4BYhlXf6jJB_KXG(k=E-U z)@_`?eB#dO<-Bnb9p&W*|D!*0Za5b`WG1QSQ_Jd+XOas{M9{56BXj4eY+7ov@4%iP z#bS5Wn+S$QO4u#zr-|$>OJgTGTG~vX`cs^It%qYxB2)X$y1mpwj^pkEZF@tZ z)vDv}zzmq*Ou^xIdbiYH8PHq-L0elqZ*X#$pv#xl7cwnE&~Ll_cAp9^F4s4E6I67e z9|}mgz$C4d0Znfzv>l6qzKTDd4M5k6W#A5=%}4dPbGI}56IEA!AEN`Av;!fj)vB$@ ztt3+yo1Se?=a)pak)%(m+I@7UK85KzDwxxKI@}o`9D0chkZv<=lnMGcPuKiuV5GsO zFi%2#op!BZPP@xQC=mc?xrg`q_*GLsk4s&R&c;dO=FcKyug~+Sv9_jo++stRhR{aW z4*w-ME(1LO9f(O!rI%5~#;ER49rr+v@NLHW2DT>C&3XNE2e z5TgW)+%&NWV|o$NAd4N-v15MF4#sLsjC+lQhlYEo-9S{xGV${NJzx@o8c}T&EBL95 zQ}f+Dk;Qi-enQ@*p8XA;JB$u0g4lpg&}HQ)g)!b*mUzpcfxrp;Nj?zR^jcglo#g1L zi}tWvGF@7maVma%d1WBm#;i=>Y_Rd%Su@kJCUm;=ub3Z~1>YYr9<-%#Zu;()NEnG1 zj=eFmkKFW7eOB8a7~Z`e{Of-R%|g&M))sew!j1oLUc1GL;9z*|=BtZgK#N*5*6cmr zVw$7>kq8`hglA@w=-y5cBP0F^BmVBabIP`hYD?fBP8j}P-gYV3pHevZUFzz<8@4H% Pgcp?L)MSgEn|$~`&9t4_ literal 0 HcmV?d00001 diff --git a/mt_with_external_memory/infer.py b/mt_with_external_memory/infer.py new file mode 100644 index 0000000000..d519ea8c40 --- /dev/null +++ b/mt_with_external_memory/infer.py @@ -0,0 +1,154 @@ +""" + Contains infering script for machine translation with external memory. +""" +import distutils.util +import argparse +import gzip +import paddle.v2 as paddle +from external_memory import ExternalMemory +from model import * +from data_utils import * + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--dict_size", + default=30000, + type=int, + help="Vocabulary size. (default: %(default)s)") +parser.add_argument( + "--word_vec_dim", + default=512, + type=int, + help="Word embedding size. (default: %(default)s)") +parser.add_argument( + "--hidden_size", + default=1024, + type=int, + help="Hidden cell number in RNN. (default: %(default)s)") +parser.add_argument( + "--memory_slot_num", + default=8, + type=int, + help="External memory slot number. (default: %(default)s)") +parser.add_argument( + "--beam_size", + default=3, + type=int, + help="Beam search width. (default: %(default)s)") +parser.add_argument( + "--use_gpu", + default=False, + type=distutils.util.strtobool, + help="Use gpu or not. (default: %(default)s)") +parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Trainer number. (default: %(default)s)") +parser.add_argument( + "--batch_size", + default=5, + type=int, + help="Batch size. (default: %(default)s)") +parser.add_argument( + "--infer_data_num", + default=3, + type=int, + help="Instance num to infer. (default: %(default)s)") +parser.add_argument( + "--model_filepath", + default="checkpoints/params.latest.tar.gz", + type=str, + help="Model filepath. (default: %(default)s)") +parser.add_argument( + "--memory_perturb_stddev", + default=0.1, + type=float, + help="Memory perturb stddev for memory initialization." + "(default: %(default)s)") +args = parser.parse_args() + + +def parse_beam_search_result(beam_result, dictionary): + """ + Beam search result parser. + """ + sentence_list = [] + sentence = [] + for word in beam_result[1]: + if word != -1: + sentence.append(word) + else: + sentence_list.append( + ' '.join([dictionary.get(word) for word in sentence[1:]])) + sentence = [] + beam_probs = beam_result[0] + beam_size = len(beam_probs[0]) + beam_sentences = [ + sentence_list[i:i + beam_size] + for i in range(0, len(sentence_list), beam_size) + ] + return beam_probs, beam_sentences + + +def infer(): + """ + For inferencing. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + beam_gen = memory_enhanced_seq2seq( + encoder_input=source_words, + decoder_input=None, + decoder_target=None, + hidden_size=args.hidden_size, + word_vec_dim=args.word_vec_dim, + dict_size=args.dict_size, + is_generating=True, + beam_size=args.beam_size) + + # load parameters + parameters = paddle.parameters.Parameters.from_tar( + gzip.open(args.model_filepath)) + + # prepare infer data + infer_data = [] + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [[ + random.gauss(0, memory_perturb_stddev) for i in xrange(args.hidden_size) + ] for j in xrange(args.memory_slot_num)] + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(dict_size), + append_tuple=(bounded_memory_perturbation, )) + for i, item in enumerate(test_append_reader()): + if i < args.infer_data_num: + infer_data.append((item[0], item[3], )) + + # run inference + beam_result = paddle.infer( + output_layer=beam_gen, + parameters=parameters, + input=infer_data, + field=['prob', 'id']) + + # parse beam result and print + source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) + beam_probs, beam_sentences = parse_beam_search_result(beam_result, + target_dict) + for i in xrange(args.infer_data_num): + print "\n***************************************************\n" + print "src:", ' '.join( + [source_dict.get(word) for word in infer_data[i][0]]), "\n" + for j in xrange(args.beam_size): + print "prob = %f : %s" % (beam_probs[i][j], beam_sentences[i][j]) + + +def main(): + paddle.init(use_gpu=False, trainer_count=1) + infer() + + +if __name__ == '__main__': + main() diff --git a/mt_with_external_memory/model.py b/mt_with_external_memory/model.py new file mode 100644 index 0000000000..6ac00d1db8 --- /dev/null +++ b/mt_with_external_memory/model.py @@ -0,0 +1,206 @@ +""" + Contains model configuration for external-memory-enhanced seq2seq. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. vanilla attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories are exploited to enhance the vanilla + Seq2Seq neural machine translation. + + The implementation primarily follows the paper + `Memory-enhanced Decoder for Neural Machine Translation + `_, + with some minor differences (will be listed in README.md). + + For details about "external memory", please also refer to + `Neural Turing Machines `_. +""" +import paddle.v2 as paddle +from external_memory import ExternalMemory + + +def bidirectional_gru_encoder(input, size, word_vec_dim): + """ + Bidirectional GRU encoder. + """ + # token embedding + embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) + # token-level forward and backard encoding for attentions + forward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=False) + backward = paddle.networks.simple_gru( + input=embeddings, size=size, reverse=True) + forward_backward = paddle.layer.concat(input=[forward, backward]) + # sequence-level encoding + backward_first = paddle.layer.first_seq(input=backward) + return forward_backward, backward_first + + +def memory_enhanced_decoder(input, target, initial_state, source_context, size, + word_vec_dim, dict_size, is_generating, beam_size): + """ + GRU sequence decoder enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this enhanced RNN decoder. + + The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the + hidden state vector (or cell state in LSTM) carrying information through + a span of sequence time, which is a successful design enriching the model + with the capability to "remember" things in the long run. However, such a + vector state is somewhat limited to a very narrow memory bandwidth. External + memory introduced here could easily increase the memory capacity with linear + complexity cost (rather than quadratic for vector state). + + This enhanced decoder expands its "memory passage" through two + ExternalMemory objects: + - Bounded memory for handling long-term information exchange within decoder + itself. A direct expansion of traditional "vector" state. + - Unbounded memory for handling source language's token-wise information. + Exactly the attention mechanism over Seq2Seq. + + Notice that we take the attention mechanism as a particular form of external + memory, with read-only memory bank initialized with encoder states, and a + read head with content-based addressing (attention). From this view point, + we arrive at a better understanding of attention mechanism itself and other + external memory, and a concise and unified implementation for them. + + For more details about external memory, please refer to + `Neural Turing Machines `_. + + For more details about this memory-enhanced decoder, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. This implementation is highly + correlated to this paper, but with minor differences (e.g. put "write" + before "read" to bypass a potential bug in V2 APIs. See + (`issue `_). + """ + # prepare initial bounded and unbounded memory + bounded_memory_slot_init = paddle.layer.fc( + input=paddle.layer.pooling( + input=source_context, pooling_type=paddle.pooling.Avg()), + size=size, + act=paddle.activation.Sigmoid()) + bounded_memory_perturbation = paddle.layer.data( + name='bounded_memory_perturbation', + type=paddle.data_type.dense_vector_sequence(size)) + bounded_memory_init = paddle.layer.addto( + input=[ + paddle.layer.expand( + input=bounded_memory_slot_init, + expand_as=bounded_memory_perturbation), + bounded_memory_perturbation + ], + act=paddle.activation.Linear()) + unbounded_memory_init = source_context + + # prepare step function for reccurent group + def recurrent_decoder_step(cur_embedding): + # create hidden state, bounded and unbounded memory. + state = paddle.layer.memory( + name="gru_decoder", size=size, boot_layer=initial_state) + bounded_memory = ExternalMemory( + name="bounded_memory", + mem_slot_size=size, + boot_layer=bounded_memory_init, + readonly=False, + enable_interpolation=True) + unbounded_memory = ExternalMemory( + name="unbounded_memory", + mem_slot_size=size * 2, + boot_layer=unbounded_memory_init, + readonly=True, + enable_interpolation=False) + # write bounded memory + bounded_memory.write(state) + # read bounded memory + bounded_memory_read = bounded_memory.read(state) + # prepare key for unbounded memory + key_for_unbounded_memory = paddle.layer.fc( + input=[bounded_memory_read, cur_embedding], + size=size, + act=paddle.activation.Tanh(), + bias_attr=False) + # read unbounded memory (i.e. attention mechanism) + context = unbounded_memory.read(key_for_unbounded_memory) + # gated recurrent unit + gru_inputs = paddle.layer.fc( + input=[context, cur_embedding, bounded_memory_read], + size=size * 3, + act=paddle.activation.Linear(), + bias_attr=False) + gru_output = paddle.layer.gru_step( + name="gru_decoder", input=gru_inputs, output_mem=state, size=size) + # step output + return paddle.layer.fc( + input=[gru_output, context, cur_embedding], + size=dict_size, + act=paddle.activation.Softmax(), + bias_attr=True) + + if not is_generating: + target_embeddings = paddle.layer.embedding( + input=input, + size=word_vec_dim, + param_attr=paddle.attr.ParamAttr(name="_decoder_word_embedding")) + decoder_result = paddle.layer.recurrent_group( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings]) + cost = paddle.layer.classification_cost( + input=decoder_result, label=target) + return cost + else: + target_embeddings = paddle.layer.GeneratedInputV2( + size=dict_size, + embedding_name="_decoder_word_embedding", + embedding_size=word_vec_dim) + beam_gen = paddle.layer.beam_search( + name="decoder_group", + step=recurrent_decoder_step, + input=[target_embeddings], + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=100) + return beam_gen + + +def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, + hidden_size, word_vec_dim, dict_size, is_generating, + beam_size): + """ + Seq2Seq Model enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this Seq2Seq model. + + Please refer to the function comments of memory_enhanced_decoder(...). + + For more details about external memory, please refer to + `Neural Turing Machines `_. + + For more details about this memory-enhanced Seq2Seq, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. + """ + # encoder + context_encodings, sequence_encoding = bidirectional_gru_encoder( + input=encoder_input, size=hidden_size, word_vec_dim=word_vec_dim) + # decoder + return memory_enhanced_decoder( + input=decoder_input, + target=decoder_target, + initial_state=sequence_encoding, + source_context=context_encodings, + size=hidden_size, + word_vec_dim=word_vec_dim, + dict_size=dict_size, + is_generating=is_generating, + beam_size=beam_size) diff --git a/mt_with_external_memory/mt_with_external_memory.py b/mt_with_external_memory/mt_with_external_memory.py deleted file mode 100644 index 22f96a72ce..0000000000 --- a/mt_with_external_memory/mt_with_external_memory.py +++ /dev/null @@ -1,598 +0,0 @@ -""" - This python script is an example model configuration for neural machine - translation with external memory, based on PaddlePaddle V2 APIs. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. vanilla attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories are exploited to enhance the vanilla - Seq2Seq neural machine translation. - - The implementation primarily follows the paper - `Memory-enhanced Decoder for Neural Machine Translation - `_, - with some minor differences (will be listed in README.md). - - For details about "external memory", please also refer to - `Neural Turing Machines `_. -""" - -import paddle.v2 as paddle -import sys -import gzip -import random - -dict_size = 30000 -word_vec_dim = 512 -hidden_size = 1024 -batch_size = 5 -memory_slot_num = 8 -beam_size = 3 -infer_data_num = 3 -memory_perturb_stddev = 0.1 - - -class ExternalMemory(object): - """ - External neural memory class. - - A simplified Neural Turing Machines (NTM) with only content-based - addressing (including content addressing and interpolation, but excluding - convolutional shift and sharpening). It serves as an external differential - memory bank, with differential write/read head controllers to store - and read information dynamically as needed. Simple feedforward networks are - used as the write/read head controllers. - - The ExternalMemory class could be utilized by many neural network structures - to easily expand their memory bandwidth and accomplish a long-term memory - handling. Besides, some existing mechanism can be realized directly with - the ExternalMemory class, e.g. the attention mechanism in Seq2Seq (i.e. an - unbounded external memory). - - Besides, the ExternalMemory class must be used together with - paddle.layer.recurrent_group (within its step function). It can never be - used in a standalone manner. - - For more details, please refer to - `Neural Turing Machines `_. - - :param name: Memory name. - :type name: basestring - :param mem_slot_size: Size of memory slot/vector. - :type mem_slot_size: int - :param boot_layer: Boot layer for initializing the external memory. The - sequence layer has sequence length indicating the number - of memory slots, and size as memory slot size. - :type boot_layer: LayerOutput - :param readonly: If true, the memory is read-only, and write function cannot - be called. Default is false. - :type readonly: bool - :param enable_interpolation: If set true, the read/write addressing weights - will be interpolated with the weights in the - last step, with the affine coefficients being - a learnable gate function. - :type enable_interpolation: bool - """ - - def __init__(self, - name, - mem_slot_size, - boot_layer, - readonly=False, - enable_interpolation=True): - self.name = name - self.mem_slot_size = mem_slot_size - self.readonly = readonly - self.enable_interpolation = enable_interpolation - self.external_memory = paddle.layer.memory( - name=self.name, - size=self.mem_slot_size, - is_seq=True, - boot_layer=boot_layer) - # prepare a constant (zero) intializer for addressing weights - self.zero_addressing_init = paddle.layer.slope_intercept( - input=paddle.layer.fc(input=boot_layer, size=1), - slope=0.0, - intercept=0.0) - # set memory to constant when readonly=True - if self.readonly: - self.updated_external_memory = paddle.layer.mixed( - name=self.name, - input=[ - paddle.layer.identity_projection(input=self.external_memory) - ], - size=self.mem_slot_size) - - def __content_addressing__(self, key_vector): - """ - Get write/read head's addressing weights via content-based addressing. - """ - # content-based addressing: a=tanh(W*M + U*key) - key_projection = paddle.layer.fc( - input=key_vector, - size=self.mem_slot_size, - act=paddle.activation.Linear(), - bias_attr=False) - key_proj_expanded = paddle.layer.expand( - input=key_projection, expand_as=self.external_memory) - memory_projection = paddle.layer.fc( - input=self.external_memory, - size=self.mem_slot_size, - act=paddle.activation.Linear(), - bias_attr=False) - merged_projection = paddle.layer.addto( - input=[key_proj_expanded, memory_projection], - act=paddle.activation.Tanh()) - # softmax addressing weight: w=softmax(v^T a) - addressing_weight = paddle.layer.fc( - input=merged_projection, - size=1, - act=paddle.activation.SequenceSoftmax(), - bias_attr=False) - return addressing_weight - - def __interpolation__(self, head_name, key_vector, addressing_weight): - """ - Interpolate between previous and current addressing weights. - """ - # prepare interpolation scalar gate: g=sigmoid(W*key) - gate = paddle.layer.fc( - input=key_vector, - size=1, - act=paddle.activation.Sigmoid(), - bias_attr=False) - # interpolation: w_t = g*w_t+(1-g)*w_{t-1} - last_addressing_weight = paddle.layer.memory( - name=self.name + "_addressing_weight_" + head_name, - size=1, - is_seq=True, - boot_layer=self.zero_addressing_init) - interpolated_weight = paddle.layer.interpolation( - name=self.name + "_addressing_weight_" + head_name, - input=[addressing_weight, addressing_weight], - weight=paddle.layer.expand(input=gate, expand_as=addressing_weight)) - return interpolated_weight - - def __get_addressing_weight__(self, head_name, key_vector): - """ - Get final addressing weights for read/write heads, including content - addressing and interpolation. - """ - # current content-based addressing - addressing_weight = self.__content_addressing__(key_vector) - # interpolation with previous addresing weight - if self.enable_interpolation: - return self.__interpolation__(head_name, key_vector, - addressing_weight) - else: - return addressing_weight - - def write(self, write_key): - """ - Write onto the external memory. - It cannot be called if "readonly" set True. - - :param write_key: Key vector for write heads to generate writing - content and addressing signals. - :type write_key: LayerOutput - """ - # check readonly - if self.readonly: - raise ValueError("ExternalMemory with readonly=True cannot write.") - # get addressing weight for write head - write_weight = self.__get_addressing_weight__("write_head", write_key) - # prepare add_vector and erase_vector - erase_vector = paddle.layer.fc( - input=write_key, - size=self.mem_slot_size, - act=paddle.activation.Sigmoid(), - bias_attr=False) - add_vector = paddle.layer.fc( - input=write_key, - size=self.mem_slot_size, - act=paddle.activation.Sigmoid(), - bias_attr=False) - erase_vector_expand = paddle.layer.expand( - input=erase_vector, expand_as=self.external_memory) - add_vector_expand = paddle.layer.expand( - input=add_vector, expand_as=self.external_memory) - # prepare scaled add part and erase part - scaled_erase_vector_expand = paddle.layer.scaling( - weight=write_weight, input=erase_vector_expand) - erase_memory_part = paddle.layer.mixed( - input=paddle.layer.dotmul_operator( - a=self.external_memory, - b=scaled_erase_vector_expand, - scale=-1.0)) - add_memory_part = paddle.layer.scaling( - weight=write_weight, input=add_vector_expand) - # update external memory - self.updated_external_memory = paddle.layer.addto( - input=[self.external_memory, add_memory_part, erase_memory_part], - name=self.name) - - def read(self, read_key): - """ - Read from the external memory. - - :param write_key: Key vector for read head to generate addressing - signals. - :type write_key: LayerOutput - :return: Content (vector) read from external memory. - :rtype: LayerOutput - """ - # get addressing weight for write head - read_weight = self.__get_addressing_weight__("read_head", read_key) - # read content from external memory - scaled = paddle.layer.scaling( - weight=read_weight, input=self.updated_external_memory) - return paddle.layer.pooling( - input=scaled, pooling_type=paddle.pooling.Sum()) - - -def bidirectional_gru_encoder(input, size, word_vec_dim): - """ - Bidirectional GRU encoder. - """ - # token embedding - embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) - # token-level forward and backard encoding for attentions - forward = paddle.networks.simple_gru( - input=embeddings, size=size, reverse=False) - backward = paddle.networks.simple_gru( - input=embeddings, size=size, reverse=True) - forward_backward = paddle.layer.concat(input=[forward, backward]) - # sequence-level encoding - backward_first = paddle.layer.first_seq(input=backward) - return forward_backward, backward_first - - -def memory_enhanced_decoder(input, target, initial_state, source_context, size, - word_vec_dim, dict_size, is_generating, beam_size): - """ - GRU sequence decoder enhanced with external memory. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories can be implemented with - ExternalMemory class, and are both exploited in this enhanced RNN decoder. - - The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the - hidden state vector (or cell state in LSTM) carrying information through - a span of sequence time, which is a successful design enriching the model - with the capability to "remember" things in the long run. However, such a - vector state is somewhat limited to a very narrow memory bandwidth. External - memory introduced here could easily increase the memory capacity with linear - complexity cost (rather than quadratic for vector state). - - This enhanced decoder expands its "memory passage" through two - ExternalMemory objects: - - Bounded memory for handling long-term information exchange within decoder - itself. A direct expansion of traditional "vector" state. - - Unbounded memory for handling source language's token-wise information. - Exactly the attention mechanism over Seq2Seq. - - Notice that we take the attention mechanism as a particular form of external - memory, with read-only memory bank initialized with encoder states, and a - read head with content-based addressing (attention). From this view point, - we arrive at a better understanding of attention mechanism itself and other - external memory, and a concise and unified implementation for them. - - For more details about external memory, please refer to - `Neural Turing Machines `_. - - For more details about this memory-enhanced decoder, please - refer to `Memory-enhanced Decoder for Neural Machine Translation - `_. This implementation is highly - correlated to this paper, but with minor differences (e.g. put "write" - before "read" to bypass a potential bug in V2 APIs. See - (`issue `_). - """ - # prepare initial bounded and unbounded memory - bounded_memory_slot_init = paddle.layer.fc( - input=paddle.layer.pooling( - input=source_context, pooling_type=paddle.pooling.Avg()), - size=size, - act=paddle.activation.Sigmoid()) - bounded_memory_perturbation = paddle.layer.data( - name='bounded_memory_perturbation', - type=paddle.data_type.dense_vector_sequence(size)) - bounded_memory_init = paddle.layer.addto( - input=[ - paddle.layer.expand( - input=bounded_memory_slot_init, - expand_as=bounded_memory_perturbation), - bounded_memory_perturbation - ], - act=paddle.activation.Linear()) - unbounded_memory_init = source_context - - # prepare step function for reccurent group - def recurrent_decoder_step(cur_embedding): - # create hidden state, bounded and unbounded memory. - state = paddle.layer.memory( - name="gru_decoder", size=size, boot_layer=initial_state) - bounded_memory = ExternalMemory( - name="bounded_memory", - mem_slot_size=size, - boot_layer=bounded_memory_init, - readonly=False, - enable_interpolation=True) - unbounded_memory = ExternalMemory( - name="unbounded_memory", - mem_slot_size=size * 2, - boot_layer=unbounded_memory_init, - readonly=True, - enable_interpolation=False) - # write bounded memory - bounded_memory.write(state) - # read bounded memory - bounded_memory_read = bounded_memory.read(state) - # prepare key for unbounded memory - key_for_unbounded_memory = paddle.layer.fc( - input=[bounded_memory_read, cur_embedding], - size=size, - act=paddle.activation.Tanh(), - bias_attr=False) - # read unbounded memory (i.e. attention mechanism) - context = unbounded_memory.read(key_for_unbounded_memory) - # gated recurrent unit - gru_inputs = paddle.layer.fc( - input=[context, cur_embedding, bounded_memory_read], - size=size * 3, - act=paddle.activation.Linear(), - bias_attr=False) - gru_output = paddle.layer.gru_step( - name="gru_decoder", input=gru_inputs, output_mem=state, size=size) - # step output - return paddle.layer.fc( - input=[gru_output, context, cur_embedding], - size=dict_size, - act=paddle.activation.Softmax(), - bias_attr=True) - - if not is_generating: - target_embeddings = paddle.layer.embedding( - input=input, - size=word_vec_dim, - param_attr=paddle.attr.ParamAttr(name="_decoder_word_embedding")) - decoder_result = paddle.layer.recurrent_group( - name="decoder_group", - step=recurrent_decoder_step, - input=[target_embeddings]) - cost = paddle.layer.classification_cost( - input=decoder_result, label=target) - return cost - else: - target_embeddings = paddle.layer.GeneratedInputV2( - size=dict_size, - embedding_name="_decoder_word_embedding", - embedding_size=word_vec_dim) - beam_gen = paddle.layer.beam_search( - name="decoder_group", - step=recurrent_decoder_step, - input=[target_embeddings], - bos_id=0, - eos_id=1, - beam_size=beam_size, - max_length=100) - return beam_gen - - -def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, - hidden_size, word_vec_dim, dict_size, is_generating, - beam_size): - """ - Seq2Seq Model enhanced with external memory. - - The "external memory" refers to two types of memories. - - Unbounded memory: i.e. attention mechanism in Seq2Seq. - - Bounded memory: i.e. external memory in NTM. - Both types of external memories can be implemented with - ExternalMemory class, and are both exploited in this Seq2Seq model. - - Please refer to the function comments of memory_enhanced_decoder(...). - - For more details about external memory, please refer to - `Neural Turing Machines `_. - - For more details about this memory-enhanced Seq2Seq, please - refer to `Memory-enhanced Decoder for Neural Machine Translation - `_. - """ - # encoder - context_encodings, sequence_encoding = bidirectional_gru_encoder( - input=encoder_input, size=hidden_size, word_vec_dim=word_vec_dim) - # decoder - return memory_enhanced_decoder( - input=decoder_input, - target=decoder_target, - initial_state=sequence_encoding, - source_context=context_encodings, - size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=is_generating, - beam_size=beam_size) - - -def parse_beam_search_result(beam_result, dictionary): - """ - Beam search result parser. - """ - sentence_list = [] - sentence = [] - for word in beam_result[1]: - if word != -1: - sentence.append(word) - else: - sentence_list.append( - ' '.join([dictionary.get(word) for word in sentence[1:]])) - sentence = [] - beam_probs = beam_result[0] - beam_size = len(beam_probs[0]) - beam_sentences = [ - sentence_list[i:i + beam_size] - for i in range(0, len(sentence_list), beam_size) - ] - return beam_probs, beam_sentences - - -def reader_append_wrapper(reader, append_tuple): - """ - Data reader wrapper for appending extra data to exisiting reader. - """ - - def new_reader(): - for ins in reader(): - yield ins + append_tuple - - return new_reader - - -def train(num_passes): - """ - For training. - """ - # create network config - source_words = paddle.layer.data( - name="source_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - target_words = paddle.layer.data( - name="target_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - target_next_words = paddle.layer.data( - name='target_next_words', - type=paddle.data_type.integer_value_sequence(dict_size)) - cost = memory_enhanced_seq2seq( - encoder_input=source_words, - decoder_input=target_words, - decoder_target=target_next_words, - hidden_size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=False, - beam_size=beam_size) - - # create parameters and optimizer - parameters = paddle.parameters.create(cost) - optimizer = paddle.optimizer.Adam( - learning_rate=5e-5, - gradient_clipping_threshold=5, - regularization=paddle.optimizer.L2Regularization(rate=8e-4)) - trainer = paddle.trainer.SGD( - cost=cost, parameters=parameters, update_equation=optimizer) - - # create data readers - feeding = { - "source_words": 0, - "target_words": 1, - "target_next_words": 2, - "bounded_memory_perturbation": 3 - } - random.seed(0) # for keeping consitancy for multiple runs - bounded_memory_perturbation = [ - [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] - for j in xrange(memory_slot_num) - ] - train_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.train(dict_size), - append_tuple=(bounded_memory_perturbation, )) - train_batch_reader = paddle.batch( - reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), - batch_size=batch_size) - test_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=(bounded_memory_perturbation, )) - test_batch_reader = paddle.batch( - reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), - batch_size=batch_size) - - # create event handler - def event_handler(event): - if isinstance(event, paddle.event.EndIteration): - if event.batch_id % 10 == 0: - print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( - event.pass_id, event.batch_id, event.cost, event.metrics) - else: - sys.stdout.write('.') - sys.stdout.flush() - if isinstance(event, paddle.event.EndPass): - result = trainer.test(reader=test_batch_reader, feeding=feeding) - print "Pass: %d, TestCost: %f, %s" % (event.pass_id, event.cost, - result.metrics) - with gzip.open("params.tar.gz", 'w') as f: - parameters.to_tar(f) - - # run train - trainer.train( - reader=train_batch_reader, - event_handler=event_handler, - num_passes=num_passes, - feeding=feeding) - - -def infer(): - """ - For inferencing. - """ - # create network config - source_words = paddle.layer.data( - name="source_words", - type=paddle.data_type.integer_value_sequence(dict_size)) - beam_gen = memory_enhanced_seq2seq( - encoder_input=source_words, - decoder_input=None, - decoder_target=None, - hidden_size=hidden_size, - word_vec_dim=word_vec_dim, - dict_size=dict_size, - is_generating=True, - beam_size=beam_size) - - # load parameters - parameters = paddle.parameters.Parameters.from_tar( - gzip.open("params.tar.gz")) - - # prepare infer data - infer_data = [] - random.seed(0) # for keeping consitancy for multiple runs - bounded_memory_perturbation = [ - [random.gauss(0, memory_perturb_stddev) for i in xrange(hidden_size)] - for j in xrange(memory_slot_num) - ] - test_append_reader = reader_append_wrapper( - reader=paddle.dataset.wmt14.test(dict_size), - append_tuple=(bounded_memory_perturbation, )) - for i, item in enumerate(test_append_reader()): - if i < infer_data_num: - infer_data.append((item[0], item[3], )) - - # run inference - beam_result = paddle.infer( - output_layer=beam_gen, - parameters=parameters, - input=infer_data, - field=['prob', 'id']) - - # parse beam result and print - source_dict, target_dict = paddle.dataset.wmt14.get_dict(dict_size) - beam_probs, beam_sentences = parse_beam_search_result(beam_result, - target_dict) - for i in xrange(infer_data_num): - print "\n***************************************************\n" - print "src:", ' '.join( - [source_dict.get(word) for word in infer_data[i][0]]), "\n" - for j in xrange(beam_size): - print "prob = %f : %s" % (beam_probs[i][j], beam_sentences[i][j]) - - -def main(): - paddle.init(use_gpu=False, trainer_count=1) - train(num_passes=1) - infer() - - -if __name__ == '__main__': - main() diff --git a/mt_with_external_memory/train.py b/mt_with_external_memory/train.py new file mode 100644 index 0000000000..91fc2750f3 --- /dev/null +++ b/mt_with_external_memory/train.py @@ -0,0 +1,157 @@ +""" + Contains training script for machine translation with external memory. +""" +import argparse +import sys +import gzip +import distutils.util +import random +import paddle.v2 as paddle +from external_memory import ExternalMemory +from model import * +from data_utils import * + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--dict_size", + default=30000, + type=int, + help="Vocabulary size. (default: %(default)s)") +parser.add_argument( + "--word_vec_dim", + default=512, + type=int, + help="Word embedding size. (default: %(default)s)") +parser.add_argument( + "--hidden_size", + default=1024, + type=int, + help="Hidden cell number in RNN. (default: %(default)s)") +parser.add_argument( + "--memory_slot_num", + default=8, + type=int, + help="External memory slot number. (default: %(default)s)") +parser.add_argument( + "--use_gpu", + default=False, + type=distutils.util.strtobool, + help="Use gpu or not. (default: %(default)s)") +parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Trainer number. (default: %(default)s)") +parser.add_argument( + "--num_passes", + default=100, + type=int, + help="Training epochs. (default: %(default)s)") +parser.add_argument( + "--batch_size", + default=5, + type=int, + help="Batch size. (default: %(default)s)") +parser.add_argument( + "--memory_perturb_stddev", + default=0.1, + type=float, + help="Memory perturb stddev for memory initialization." + "(default: %(default)s)") +args = parser.parse_args() + + +def train(): + """ + For training. + """ + # create network config + source_words = paddle.layer.data( + name="source_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + target_words = paddle.layer.data( + name="target_words", + type=paddle.data_type.integer_value_sequence(args.dict_size)) + target_next_words = paddle.layer.data( + name='target_next_words', + type=paddle.data_type.integer_value_sequence(args.dict_size)) + cost = memory_enhanced_seq2seq( + encoder_input=source_words, + decoder_input=target_words, + decoder_target=target_next_words, + hidden_size=args.hidden_size, + word_vec_dim=args.word_vec_dim, + dict_size=args.dict_size, + is_generating=False, + beam_size=None) + + # create parameters and optimizer + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Adam( + learning_rate=5e-5, + gradient_clipping_threshold=5, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + trainer = paddle.trainer.SGD( + cost=cost, parameters=parameters, update_equation=optimizer) + + # create data readers + feeding = { + "source_words": 0, + "target_words": 1, + "target_next_words": 2, + "bounded_memory_perturbation": 3 + } + random.seed(0) # for keeping consitancy for multiple runs + bounded_memory_perturbation = [[ + random.gauss(0, args.memory_perturb_stddev) + for i in xrange(args.hidden_size) + ] for j in xrange(args.memory_slot_num)] + train_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.train(args.dict_size), + append_tuple=(bounded_memory_perturbation, )) + train_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=train_append_reader, buf_size=8192), + batch_size=args.batch_size) + test_append_reader = reader_append_wrapper( + reader=paddle.dataset.wmt14.test(args.dict_size), + append_tuple=(bounded_memory_perturbation, )) + test_batch_reader = paddle.batch( + reader=paddle.reader.shuffle(reader=test_append_reader, buf_size=8192), + batch_size=args.batch_size) + + # create event handler + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 10 == 0: + print "Pass: %d, Batch: %d, TrainCost: %f, %s" % ( + event.pass_id, event.batch_id, event.cost, event.metrics) + with gzip.open("checkpoints/params.latest.tar.gz", 'w') as f: + parameters.to_tar(f) + else: + sys.stdout.write('.') + sys.stdout.flush() + if isinstance(event, paddle.event.EndPass): + result = trainer.test(reader=test_batch_reader, feeding=feeding) + print "Pass: %d, TestCost: %f, %s" % (event.pass_id, event.cost, + result.metrics) + with gzip.open("checkpoints/params.pass-%d.tar.gz" % event.pass_id, + 'w') as f: + parameters.to_tar(f) + + # run train + if not os.path.exists('checkpoints'): + os.mkdir('checkpoints') + trainer.train( + reader=train_batch_reader, + event_handler=event_handler, + num_passes=args.num_passes, + feeding=feeding) + + +def main(): + paddle.init(use_gpu=args.use_gpu, trainer_count=args.trainer_count) + train() + + +if __name__ == '__main__': + main() From 69b4bdadb258387df67913bb6479839984822897 Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Thu, 13 Jul 2017 16:46:45 +0800 Subject: [PATCH 8/9] Further update README.md and add function doc for mt_with_external_memory model. --- mt_with_external_memory/README.md | 190 +++++++++++++++++++-- mt_with_external_memory/external_memory.py | 18 +- mt_with_external_memory/model.py | 62 ++++++- 3 files changed, 236 insertions(+), 34 deletions(-) diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 22c8effdcd..96dfff0d4f 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -222,7 +222,7 @@ class ExternalMemory(object): - 输入参数 `read_key`：某层的输出，其包含的信息用于读头的寻址。 - 返回：读出的信息（可直接作为其他层的输入）。 -部分重要的实现逻辑： +部分关键实现逻辑： - 神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，并采用序列形式（`is_seq=True`），该序列的长度表示记忆槽的数量，序列的 `size` 表示记忆槽（向量）的大小。该序列依赖一个外部层作为初始化， 其记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 @@ -244,21 +244,130 @@ class ExternalMemory(object): 涉及三个主要函数： ``` -memory_enhanced_seq2seq(...) -bidirectional_gru_encoder(...) -memory_enhanced_decoder(...) +def bidirectional_gru_encoder(input, size, word_vec_dim): + """Bidirectional GRU encoder. + + :params size: Hidden cell number in decoder rnn. + :type size: int + :params word_vec_dim: Word embedding size. + :type word_vec_dim: int + :return: Tuple of 1. concatenated forward and backward hidden sequence. + 2. last state of backward rnn. + :rtype: tuple of LayerOutput + """ + pass + + +def memory_enhanced_decoder(input, target, initial_state, source_context, size, + word_vec_dim, dict_size, is_generating, beam_size): + """GRU sequence decoder enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this enhanced RNN decoder. + + The vanilla RNN/LSTM/GRU also has a narrow memory mechanism, namely the + hidden state vector (or cell state in LSTM) carrying information through + a span of sequence time, which is a successful design enriching the model + with the capability to "remember" things in the long run. However, such a + vector state is somewhat limited to a very narrow memory bandwidth. External + memory introduced here could easily increase the memory capacity with linear + complexity cost (rather than quadratic for vector state). + + This enhanced decoder expands its "memory passage" through two + ExternalMemory objects: + - Bounded memory for handling long-term information exchange within decoder + itself. A direct expansion of traditional "vector" state. + - Unbounded memory for handling source language's token-wise information. + Exactly the attention mechanism over Seq2Seq. + + Notice that we take the attention mechanism as a particular form of external + memory, with read-only memory bank initialized with encoder states, and a + read head with content-based addressing (attention). From this view point, + we arrive at a better understanding of attention mechanism itself and other + external memory, and a concise and unified implementation for them. + + For more details about external memory, please refer to + `Neural Turing Machines `_. + + For more details about this memory-enhanced decoder, please + refer to `Memory-enhanced Decoder for Neural Machine Translation + `_. This implementation is highly + correlated to this paper, but with minor differences (e.g. put "write" + before "read" to bypass a potential bug in V2 APIs. See + (`issue `_). + """ + pass + + + +def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, + hidden_size, word_vec_dim, dict_size, is_generating, + beam_size): + """Seq2Seq Model enhanced with external memory. + + The "external memory" refers to two types of memories. + - Unbounded memory: i.e. attention mechanism in Seq2Seq. + - Bounded memory: i.e. external memory in NTM. + Both types of external memories can be implemented with + ExternalMemory class, and are both exploited in this Seq2Seq model. + + :params encoder_input: Encoder input. + :type encoder_input: LayerOutput + :params decoder_input: Decoder input. + :type decoder_input: LayerOutput + :params decoder_target: Decoder target. + :type decoder_target: LayerOutput + :params hidden_size: Hidden cell number, both in encoder and decoder rnn. + :type hidden_size: int + :params word_vec_dim: Word embedding size. + :type word_vec_dim: int + :param dict_size: Vocabulary size. + :type dict_size: int + :params is_generating: Whether for beam search inferencing (True) or + for training (False). + :type is_generating: bool + :params beam_size: Beam search width. + :type beam_size: int + :return: Cost layer if is_generating=False; Beam search layer if + is_generating = True. + :rtype: LayerOutput + """ + pass ``` -`memory_enhanced_seq2seq` 函数定义整个带外部记忆机制的序列到序列模型，是模型定义的主调函数。它首先调用`bidirectional_gru_encoder` 对源语言进行编码，然后通过 `memory_enhanced_decoder` 进行解码。 +- `bidirectional_gru_encoder` 函数实现双向单层 GRU（Gated Recurrent Unit） 编码器。返回两组结果：一组为字符级编码向量序列（包含前后向），一组为整个源语句的句级编码向量（仅后向）。前者用于解码器的注意力机制中记忆矩阵的初始化，后者用于解码器的状态向量的初始化。 + +- `memory_enhanced_decoder` 函数实现通过外部记忆增强的 GRU 解码器。它利用同一个`ExternalMemory` 类实现两种外部记忆模块： + + - 无界外部记忆：即传统的注意力机制。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符数。 + + ``` + unbounded_memory = ExternalMemory( + name="unbounded_memory", + mem_slot_size=size * 2, + boot_layer=unbounded_memory_init, + readonly=True, + enable_interpolation=False) + ``` + - 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值池化（pooling）后并扩展为指定序列长度后，叠加随机噪声（训练和推断时保持一致），作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是固定的。即代码中的： -`bidirectional_gru_encoder` 函数实现双向单层 GRU（Gated Recurrent Unit） 编码器。返回两组结果：一组为字符级编码向量序列（包含前后向），一组为整个源语句的句级编码向量（仅后向）。前者用于解码器的注意力机制中记忆矩阵的初始化，后者用于解码器的状态向量的初始化。 + ``` + bounded_memory = ExternalMemory( + name="bounded_memory", + mem_slot_size=size, + boot_layer=bounded_memory_init, + readonly=False, + enable_interpolation=True) + ``` -`memory_enhanced_decoder` 函数实现通过外部记忆增强的 GRU 解码器。它利用同一个`ExternalMemory` 类实现两种外部记忆模块： + 注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 `ExternalMemory` 类。前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “注意机制” 和 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 + +- `memory_enhanced_seq2seq` 函数定义整个带外部记忆机制的序列到序列模型，是模型定义的主调函数。它首先调用`bidirectional_gru_encoder` 对源语言进行编码，然后通过 `memory_enhanced_decoder` 进行解码。 -- 无界外部记忆：即传统的注意力机制。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符数。 -- 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值池化（pooling）后并扩展为指定序列长度后，叠加随机噪声（训练和推断时保持一致），作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是固定的。 -注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 `ExternalMemory` 类。前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “注意机制” 和 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 此外，在该实现中，将 `ExternalMemory` 的 `write` 操作提前至 `read` 之前，以避开潜在的拓扑连接局限，详见 [Issue](https://github.com/PaddlePaddle/Paddle/issues/2061)。我们可以看到，本质上他们是等价的。 @@ -278,24 +387,73 @@ def reader(): 用户需自行完成字符的切分 (Tokenize) ，并构建字典完成 ID 化。 -PaddlePaddle 的接口 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)， 默认提供了一个经过预处理的、较小规模的 wmt14 英法翻译数据集的子集。并提供了两个reader creator函数如下： +PaddlePaddle 的接口 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)， 默认提供了一个经过预处理的、较小规模的 [wmt14 英法翻译数据集的子集](http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz)(该数据集有193319条训练数据，6003条测试数据，词典长度为30000)。并提供了两个reader creator函数如下： ``` paddle.dataset.wmt14.train(dict_size) paddle.dataset.wmt14.test(dict_size) ``` -这两个函数被调用时即返回相应的`reader()`函数，供`paddle.traner.SGD.train`使用。 +这两个函数被调用时即返回相应的`reader()`函数，供`paddle.traner.SGD.train`使用。当我们需要使用其他数据时，可参考 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py) 构造相应的 data creator，并替换 `paddle.dataset.wmt14.train` 和 `paddle.dataset.wmt14.train` 成相应函数名。 + +### 训练 + +命令行输入： -当我们需要使用其他数据时，可参考 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py) 构造相应的 data creator，并替换 `paddle.dataset.wmt14.train` 和 `paddle.dataset.wmt14.train` 成相应函数名。 +``` +python mt_with_external_memory.py +``` +或自定义部分参数, 例如: -### 训练及预测 +``` +CUDA_VISIBLE_DEVICES=8,9,10,11 python train.py \ + --dict_size 30000 \ + --word_vec_dim 512 \ + --hidden_size 1024 \ + --memory_slot_num 8 \ + --use_gpu True \ + --trainer_count 4 \ + --num_passes 100 \ + --batch_size 128 \ + --memory_perturb_stddev 0.1 +``` + +即可运行训练脚本，训练模型将被定期保存于本地 `./checkpoints`。参数含义可运行 + +``` +python train.py --help +``` + + +### 解码 命令行输入： -```python mt_with_external_memory.py``` +``` +python infer.py +``` +或自定义部分参数, 例如: + +``` +CUDA_VISIBLE_DEVICES=8,9,10,11 python train.py \ + --dict_size 30000 \ + --word_vec_dim 512 \ + --hidden_size 1024 \ + --memory_slot_num 8 \ + --use_gpu True \ + --trainer_count 4 \ + --memory_perturb_stddev 0.1 \ + --infer_num_data 10 \ + --model_filepath checkpoints/params.latest.tar.gz + --beam_size 3 +``` + +即可运行解码脚本，产生示例翻译结果。参数含义可运行： + +``` +python infer.py --help +``` -即可运行训练脚本（默认训练一轮），训练模型将被定期保存于本地 `params.tar.gz`。训练完成后，将为少量样本生成翻译结果，详见 `infer` 函数。 ## 其他讨论 diff --git a/mt_with_external_memory/external_memory.py b/mt_with_external_memory/external_memory.py index d3dbf32ff3..f8597b7b2a 100755 --- a/mt_with_external_memory/external_memory.py +++ b/mt_with_external_memory/external_memory.py @@ -5,8 +5,7 @@ class ExternalMemory(object): - """ - External neural memory class. + """External neural memory class. A simplified Neural Turing Machines (NTM) with only content-based addressing (including content addressing and interpolation, but excluding @@ -76,8 +75,7 @@ def __init__(self, size=self.mem_slot_size) def _content_addressing(self, key_vector): - """ - Get write/read head's addressing weights via content-based addressing. + """Get write/read head's addressing weights via content-based addressing. """ # content-based addressing: a=tanh(W*M + U*key) key_projection = paddle.layer.fc( @@ -104,8 +102,7 @@ def _content_addressing(self, key_vector): return addressing_weight def _interpolation(self, head_name, key_vector, addressing_weight): - """ - Interpolate between previous and current addressing weights. + """Interpolate between previous and current addressing weights. """ # prepare interpolation scalar gate: g=sigmoid(W*key) gate = paddle.layer.fc( @@ -126,8 +123,7 @@ def _interpolation(self, head_name, key_vector, addressing_weight): return interpolated_weight def _get_addressing_weight(self, head_name, key_vector): - """ - Get final addressing weights for read/write heads, including content + """Get final addressing weights for read/write heads, including content addressing and interpolation. """ # current content-based addressing @@ -139,8 +135,7 @@ def _get_addressing_weight(self, head_name, key_vector): return addressing_weight def write(self, write_key): - """ - Write onto the external memory. + """Write onto the external memory. It cannot be called if "readonly" set True. :param write_key: Key vector for write heads to generate writing @@ -183,8 +178,7 @@ def write(self, write_key): name=self.name) def read(self, read_key): - """ - Read from the external memory. + """Read from the external memory. :param write_key: Key vector for read head to generate addressing signals. diff --git a/mt_with_external_memory/model.py b/mt_with_external_memory/model.py index 6ac00d1db8..af8414a3fa 100644 --- a/mt_with_external_memory/model.py +++ b/mt_with_external_memory/model.py @@ -20,8 +20,15 @@ def bidirectional_gru_encoder(input, size, word_vec_dim): - """ - Bidirectional GRU encoder. + """Bidirectional GRU encoder. + + :params size: Hidden cell number in decoder rnn. + :type size: int + :params word_vec_dim: Word embedding size. + :type word_vec_dim: int + :return: Tuple of 1. concatenated forward and backward hidden sequence. + 2. last state of backward rnn. + :rtype: tuple of LayerOutput """ # token embedding embeddings = paddle.layer.embedding(input=input, size=word_vec_dim) @@ -38,8 +45,7 @@ def bidirectional_gru_encoder(input, size, word_vec_dim): def memory_enhanced_decoder(input, target, initial_state, source_context, size, word_vec_dim, dict_size, is_generating, beam_size): - """ - GRU sequence decoder enhanced with external memory. + """GRU sequence decoder enhanced with external memory. The "external memory" refers to two types of memories. - Unbounded memory: i.e. attention mechanism in Seq2Seq. @@ -77,6 +83,30 @@ def memory_enhanced_decoder(input, target, initial_state, source_context, size, correlated to this paper, but with minor differences (e.g. put "write" before "read" to bypass a potential bug in V2 APIs. See (`issue `_). + + :params input: Decoder input. + :type input: LayerOutput + :params target: Decoder target. + :type target: LayerOutput + :params initial_state: Initial hidden state. + :type initial_state: LayerOutput + :params source_context: Group of context hidden states for each token in the + source sentence, for attention mechanisim. + :type source_context: LayerOutput + :params size: Hidden cell number in decoder rnn. + :type size: int + :params word_vec_dim: Word embedding size. + :type word_vec_dim: int + :param dict_size: Vocabulary size. + :type dict_size: int + :params is_generating: Whether for beam search inferencing (True) or + for training (False). + :type is_generating: bool + :params beam_size: Beam search width. + :type beam_size: int + :return: Cost layer if is_generating=False; Beam search layer if + is_generating = True. + :rtype: LayerOutput """ # prepare initial bounded and unbounded memory bounded_memory_slot_init = paddle.layer.fc( @@ -172,8 +202,7 @@ def recurrent_decoder_step(cur_embedding): def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, hidden_size, word_vec_dim, dict_size, is_generating, beam_size): - """ - Seq2Seq Model enhanced with external memory. + """Seq2Seq Model enhanced with external memory. The "external memory" refers to two types of memories. - Unbounded memory: i.e. attention mechanism in Seq2Seq. @@ -189,6 +218,27 @@ def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, For more details about this memory-enhanced Seq2Seq, please refer to `Memory-enhanced Decoder for Neural Machine Translation `_. + + :params encoder_input: Encoder input. + :type encoder_input: LayerOutput + :params decoder_input: Decoder input. + :type decoder_input: LayerOutput + :params decoder_target: Decoder target. + :type decoder_target: LayerOutput + :params hidden_size: Hidden cell number, both in encoder and decoder rnn. + :type hidden_size: int + :params word_vec_dim: Word embedding size. + :type word_vec_dim: int + :param dict_size: Vocabulary size. + :type dict_size: int + :params is_generating: Whether for beam search inferencing (True) or + for training (False). + :type is_generating: bool + :params beam_size: Beam search width. + :type beam_size: int + :return: Cost layer if is_generating=False; Beam search layer if + is_generating = True. + :rtype: LayerOutput """ # encoder context_encodings, sequence_encoding = bidirectional_gru_encoder( From 54afcbc4028e385fc2daa9eae7cc3bf0f9ee51ff Mon Sep 17 00:00:00 2001 From: Xinghai Sun Date: Tue, 12 Sep 2017 20:58:06 +0800 Subject: [PATCH 9/9] Update by following reviewer's comments. --- mt_with_external_memory/README.md | 110 ++++++++++----------- mt_with_external_memory/external_memory.py | 6 +- mt_with_external_memory/infer.py | 5 +- mt_with_external_memory/model.py | 2 +- mt_with_external_memory/train.py | 17 ++-- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/mt_with_external_memory/README.md b/mt_with_external_memory/README.md index 96dfff0d4f..266ebbf961 100644 --- a/mt_with_external_memory/README.md +++ b/mt_with_external_memory/README.md @@ -32,7 +32,7 @@ #### 动态记忆 2 --- Seq2Seq 中的注意力机制 -然而上节所属的单个向量 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着较大的潜在信息丢失。 +然而上节所述的单个向量 $h$ 或 $c$ 的信息带宽有限。在序列到序列生成模型中，这样的带宽瓶颈更表现在信息从编码器（Encoder）转移至解码器（Decoder）的过程中：仅仅依赖一个有限长度的状态向量来编码整个变长的源语句，有着较大的潜在信息丢失。 \[[3](#参考文献)\] 提出了注意力机制（Attention Mechanism），以克服上述困难。在解码时，解码器不再仅仅依赖来自编码器的唯一的句级编码向量的信息，而是依赖一个向量组的记忆信息：向量组中的每个向量为编码器的各字符（Token）的编码向量（例如 $h_t$）。通过一组可学习的注意强度（Attention Weights) 来动态分配注意力资源，以线性加权方式读取信息，用于序列的不同时间步的符号生成（可参考 PaddlePaddle Book [机器翻译](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)一章）。这种注意强度的分布，可看成基于内容的寻址（请参考神经图灵机 \[[1](#参考文献)\] 中的寻址描述），即在源语句的不同位置根据其内容决定不同的读取强度，起到一种和源语句 “软对齐（Soft Alignment）” 的作用。 @@ -40,7 +40,7 @@ #### 动态记忆 3 --- 神经图灵机 -图灵机（Turing Machines）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模拟可微分（即可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式的动态存储。神经图灵机正是要弥补这样的潜在缺陷。 +图灵机（Turing Machine）或冯诺依曼体系（Von Neumann Architecture），是计算机体系结构的雏形。运算器（如代数计算）、控制器（如逻辑分支控制）和存储器三者一体，共同构成了当代计算机的核心运行机制。神经图灵机（Neural Turing Machines）\[[1](#参考文献)\] 试图利用神经网络模拟可微分（即可通过梯度下降来学习）的图灵机，以实现更复杂的智能。而一般的机器学习模型，大部分忽略了显式的动态存储。神经图灵机正是要弥补这样的潜在缺陷。

@@ -123,7 +123,7 @@ 该类结构如下： -``` +```python class ExternalMemory(object): """External neural memory class. @@ -214,7 +214,7 @@ class ExternalMemory(object): - 输入参数 `name`: 外部记忆单元名，不同实例的相同命名将共享同一外部记忆单元。 - 输入参数 `mem_slot_size`: 单个记忆槽（向量）的维度。 - 输入参数 `boot_layer`: 用于内存槽初始化的层。需为序列类型，序列长度表明记忆槽的数量。 - - 输入参数 `readonly`: 是否打开只读模式（例如打开只读模式，该实例可用于注意力机制）。打开是，`write` 方法不可被调用。 + - 输入参数 `readonly`: 是否打开只读模式（例如打开只读模式，该实例可用于注意力机制）。打开只读模式，`write` 方法不可被调用。 - 输入参数 `enable_interpolation`: 是否允许插值寻址（例如当用于注意力机制时，需要关闭插值寻址）。 - `write`: 写操作。 - 输入参数 `write_key`：某层的输出，其包含的信息用于写头的寻址和实际写入信息的生成。 @@ -224,14 +224,14 @@ class ExternalMemory(object): 部分关键实现逻辑： -- 神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现，并采用序列形式（`is_seq=True`），该序列的长度表示记忆槽的数量，序列的 `size` 表示记忆槽（向量）的大小。该序列依赖一个外部层作为初始化， 其记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 +- 神经图灵机的 “外部存储矩阵” 采用 `Paddle.layer.memory`实现。该序列的长度表示记忆槽的数量，序列的 `size` 表示记忆槽（向量）的大小。该序列依赖一个外部层作为初始化， 其记忆槽的数量取决于该层输出序列的长度。因此，该类不仅可用来实现有界记忆（Bounded Memory)，同时可用来实现无界记忆 (Unbounded Memory，即记忆槽数量可变)。 - ``` - self.external_memory = paddle.layer.memory( - name=self.name, - size=self.mem_slot_size, - is_seq=True, - boot_layer=boot_layer) + ```python + self.external_memory = paddle.layer.memory( + name=self.name, + size=self.mem_slot_size, + is_seq=True, + boot_layer=boot_layer) ``` - `ExternalMemory`类的寻址逻辑通过 `_content_addressing` 和 `_interpolation` 两个私有方法实现。读和写操作通过 `read` 和 `write` 两个函数实现，包括上述的寻址操作。并且读和写的寻址独立进行，不同于 \[[2](#参考文献)\] 中的二者共享同一个寻址强度，目的是为了使得该类更通用。 - 为了简单起见，控制器（Controller）未被专门模块化，而是分散在各个寻址和读写函数中。控制器主要包括寻址操作和写操作时生成写入/擦除向量等，其中寻址操作通过上述的`_content_addressing` 和 `_interpolation` 两个私有方法实现，写操作时的写入/擦除向量的生成则在 `write` 方法中实现。上述均采用简单的前馈网络模拟控制器。读者可尝试剥离控制器逻辑并模块化，同时可尝试循环神经网络做控制器。 @@ -243,7 +243,7 @@ class ExternalMemory(object): 涉及三个主要函数： -``` +```python def bidirectional_gru_encoder(input, size, word_vec_dim): """Bidirectional GRU encoder. @@ -344,23 +344,23 @@ def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, - 无界外部记忆：即传统的注意力机制。利用`ExternalMemory`，打开只读开关，关闭插值寻址。并利用解码器的第一组输出作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是动态可变的，取决于编码器的字符数。 - ``` - unbounded_memory = ExternalMemory( - name="unbounded_memory", - mem_slot_size=size * 2, - boot_layer=unbounded_memory_init, - readonly=True, - enable_interpolation=False) + ```python + unbounded_memory = ExternalMemory( + name="unbounded_memory", + mem_slot_size=size * 2, + boot_layer=unbounded_memory_init, + readonly=True, + enable_interpolation=False) ``` - 有界外部记忆：利用`ExternalMemory`，关闭只读开关，打开插值寻址。并利用解码器的第一组输出，取均值池化（pooling）后并扩展为指定序列长度后，叠加随机噪声（训练和推断时保持一致），作为 `ExternalMemory` 中存储矩阵的初始化（`boot_layer`）。因此，该存储的记忆槽数目是固定的。即代码中的： - ``` - bounded_memory = ExternalMemory( - name="bounded_memory", - mem_slot_size=size, - boot_layer=bounded_memory_init, - readonly=False, - enable_interpolation=True) + ```python + bounded_memory = ExternalMemory( + name="bounded_memory", + mem_slot_size=size, + boot_layer=bounded_memory_init, + readonly=False, + enable_interpolation=True) ``` 注意到，在我们的实现中，注意力机制（或无界外部存储）和神经图灵机（或有界外部存储）被实现成相同的 `ExternalMemory` 类。前者是**只读**的， 后者**可读可写**。这样处理仅仅是为了便于统一我们对 “注意机制” 和 “记忆机制” 的理解和认识，同时也提供更简洁和统一的实现版本。注意力机制也可以通过 `paddle.networks.simple_attention` 实现。 @@ -377,7 +377,7 @@ def memory_enhanced_seq2seq(encoder_input, decoder_input, decoder_target, 数据是通过无参的 `reader()` 迭代器函数，进入训练过程。因此我们需要为训练数据和测试数据分别构造两个 `reader()` 迭代器。`reader()` 函数使用 `yield` 来实现迭代器功能（即可通过 `for instance in reader()` 方式迭代运行）， 例如 -``` +```python def reader(): for instance in data_list: yield instance @@ -389,7 +389,7 @@ def reader(): PaddlePaddle 的接口 [paddle.paddle.wmt14](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/dataset/wmt14.py)， 默认提供了一个经过预处理的、较小规模的 [wmt14 英法翻译数据集的子集](http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz)(该数据集有193319条训练数据，6003条测试数据，词典长度为30000)。并提供了两个reader creator函数如下： -``` +```python paddle.dataset.wmt14.train(dict_size) paddle.dataset.wmt14.test(dict_size) ``` @@ -400,27 +400,27 @@ paddle.dataset.wmt14.test(dict_size) 命令行输入： -``` +```bash python mt_with_external_memory.py ``` 或自定义部分参数, 例如: -``` -CUDA_VISIBLE_DEVICES=8,9,10,11 python train.py \ - --dict_size 30000 \ - --word_vec_dim 512 \ - --hidden_size 1024 \ - --memory_slot_num 8 \ - --use_gpu True \ - --trainer_count 4 \ - --num_passes 100 \ - --batch_size 128 \ - --memory_perturb_stddev 0.1 +```bash +python train.py \ +--dict_size 30000 \ +--word_vec_dim 512 \ +--hidden_size 1024 \ +--memory_slot_num 8 \ +--use_gpu False \ +--trainer_count 1 \ +--num_passes 100 \ +--batch_size 128 \ +--memory_perturb_stddev 0.1 ``` 即可运行训练脚本，训练模型将被定期保存于本地 `./checkpoints`。参数含义可运行 -``` +```bash python train.py --help ``` @@ -429,28 +429,28 @@ python train.py --help 命令行输入： -``` +```bash python infer.py ``` 或自定义部分参数, 例如: -``` -CUDA_VISIBLE_DEVICES=8,9,10,11 python train.py \ - --dict_size 30000 \ - --word_vec_dim 512 \ - --hidden_size 1024 \ - --memory_slot_num 8 \ - --use_gpu True \ - --trainer_count 4 \ - --memory_perturb_stddev 0.1 \ - --infer_num_data 10 \ - --model_filepath checkpoints/params.latest.tar.gz - --beam_size 3 +```bash +python train.py \ +--dict_size 30000 \ +--word_vec_dim 512 \ +--hidden_size 1024 \ +--memory_slot_num 8 \ +--use_gpu False \ +--trainer_count 1 \ +--memory_perturb_stddev 0.1 \ +--infer_num_data 10 \ +--model_filepath checkpoints/params.latest.tar.gz \ +--beam_size 3 ``` 即可运行解码脚本，产生示例翻译结果。参数含义可运行： -``` +```bash python infer.py --help ``` diff --git a/mt_with_external_memory/external_memory.py b/mt_with_external_memory/external_memory.py index f8597b7b2a..84eebf0216 100755 --- a/mt_with_external_memory/external_memory.py +++ b/mt_with_external_memory/external_memory.py @@ -56,10 +56,7 @@ def __init__(self, self.readonly = readonly self.enable_interpolation = enable_interpolation self.external_memory = paddle.layer.memory( - name=self.name, - size=self.mem_slot_size, - is_seq=True, - boot_layer=boot_layer) + name=self.name, size=self.mem_slot_size, boot_layer=boot_layer) # prepare a constant (zero) intializer for addressing weights self.zero_addressing_init = paddle.layer.slope_intercept( input=paddle.layer.fc(input=boot_layer, size=1), @@ -114,7 +111,6 @@ def _interpolation(self, head_name, key_vector, addressing_weight): last_addressing_weight = paddle.layer.memory( name=self.name + "_addressing_weight_" + head_name, size=1, - is_seq=True, boot_layer=self.zero_addressing_init) interpolated_weight = paddle.layer.interpolation( name=self.name + "_addressing_weight_" + head_name, diff --git a/mt_with_external_memory/infer.py b/mt_with_external_memory/infer.py index d519ea8c40..063fb24f98 100644 --- a/mt_with_external_memory/infer.py +++ b/mt_with_external_memory/infer.py @@ -4,10 +4,11 @@ import distutils.util import argparse import gzip + import paddle.v2 as paddle from external_memory import ExternalMemory -from model import * -from data_utils import * +from model import memory_enhanced_seq2seq +from data_utils import reader_append_wrapper parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/mt_with_external_memory/model.py b/mt_with_external_memory/model.py index af8414a3fa..675671025b 100644 --- a/mt_with_external_memory/model.py +++ b/mt_with_external_memory/model.py @@ -184,7 +184,7 @@ def recurrent_decoder_step(cur_embedding): input=decoder_result, label=target) return cost else: - target_embeddings = paddle.layer.GeneratedInputV2( + target_embeddings = paddle.layer.GeneratedInput( size=dict_size, embedding_name="_decoder_word_embedding", embedding_size=word_vec_dim) diff --git a/mt_with_external_memory/train.py b/mt_with_external_memory/train.py index 91fc2750f3..6dd9187150 100644 --- a/mt_with_external_memory/train.py +++ b/mt_with_external_memory/train.py @@ -6,10 +6,11 @@ import gzip import distutils.util import random + import paddle.v2 as paddle from external_memory import ExternalMemory -from model import * -from data_utils import * +from model import memory_enhanced_seq2seq +from data_utils import reader_append_wrapper parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -65,6 +66,12 @@ def train(): """ For training. """ + # create optimizer + optimizer = paddle.optimizer.Adam( + learning_rate=5e-5, + gradient_clipping_threshold=5, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + # create network config source_words = paddle.layer.data( name="source_words", @@ -85,12 +92,8 @@ def train(): is_generating=False, beam_size=None) - # create parameters and optimizer + # create parameters and trainer parameters = paddle.parameters.create(cost) - optimizer = paddle.optimizer.Adam( - learning_rate=5e-5, - gradient_clipping_threshold=5, - regularization=paddle.optimizer.L2Regularization(rate=8e-4)) trainer = paddle.trainer.SGD( cost=cost, parameters=parameters, update_equation=optimizer)