Transformer.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"XFG0NDRu5mYQ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721906553584,"user_tz":-540,"elapsed":111031,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"818ae018-0cca-478a-a88e-458afb8e6364"},"outputs":[{"output_type":"stream","name":"stdout","text":["Reading package lists... Done\n","Building dependency tree... Done\n","Reading state information... Done\n","Package libcudnn8 is not available, but is referred to by another package.\n","This may mean that the package is missing, has been obsoleted, or\n","is only available from another source\n","\n","\u001b[1;31mE: \u001b[0mVersion '8.1.0.77-1+cuda11.2' for 'libcudnn8' was not found\u001b[0m\n","\u001b[33mWARNING: Skipping tensorflow-text as it is not installed.\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: protobuf~=3.20.3 in /usr/local/lib/python3.10/dist-packages (3.20.3)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.8/47.8 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m72.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m601.3/601.3 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.3/5.3 MB\u001b[0m \u001b[31m90.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m46.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m59.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m87.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m347.7/347.7 kB\u001b[0m \u001b[31m22.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","tf-keras 2.15.1 requires tensorflow<2.16,>=2.15, but you have tensorflow 2.17.0 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0m"]}],"source":["!apt install --allow-change-held-packages libcudnn8=8.1.0.77-1+cuda11.2\n","!pip uninstall -y -q tensorflow keras tensorflow-estimator tensorflow-text\n","!pip install protobuf~=3.20.3\n","!pip install -q tensorflow_datasets\n","!pip install -q -U tensorflow-text tensorflow"]},{"cell_type":"markdown","source":["# 라이브러리"],"metadata":{"id":"vBHNHuFZ00gK"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"JjJJyJTZYebt"},"outputs":[],"source":["# 파이썬 서드파티 라이브러리\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import tensorflow_datasets as tfds\n","import tensorflow_text # tf.saved_model.load로 dataset의 tokenizer를 불러올 때 필요\n","import tensorflow as tf"]},{"cell_type":"markdown","source":["# 클래스"],"metadata":{"id":"1EH2V1YA1e_h"}},{"cell_type":"markdown","source":["## 위치 임베딩"],"metadata":{"id":"lldnaORI2WZh"}},{"cell_type":"code","source":["class PositionalEmbedding(tf.keras.layers.Layer):\n","    @staticmethod\n","    def positional_encoding(length: int, depth: int) -> tf.Tensor:\n","        \"\"\"고정된 위치 인코딩 결과값을 출력하는 함수\n","\n","        Args:\n","            length: 입력 시퀀스의 길이 -> int\n","            depth: 출력 크기\n","\n","        Returns:\n","            tf.Tensor\n","        \"\"\"\n","        depth = depth / 2\n","\n","        positions = np.arange(length)[:, np.newaxis]     # (seq, 1)\n","        depths = np.arange(depth)[np.newaxis, :] / depth   # (1, depth)\n","\n","        angle_rates = 1 / (10000**depths)         # (1, depth)\n","        angle_rads = positions * angle_rates      # (pos, depth)\n","\n","        pos_encoding = np.concatenate([np.sin(angle_rads), np.cos(angle_rads)], axis=-1)\n","\n","        return tf.cast(pos_encoding, dtype=tf.float32)\n","\n","    def __init__(self, vocab_size, d_model):\n","        super().__init__()\n","        self.d_model = d_model\n","        self.embedding = tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=d_model, mask_zero=True)\n","        self.pos_encoding = PositionalEmbedding.positional_encoding(length=2048, depth=d_model)\n","\n","    def compute_mask(self, *args, **kwargs):\n","        return self.embedding.compute_mask(*args, **kwargs)\n","\n","    def call(self, x):\n","        length = tf.shape(x)[1]\n","        x = self.embedding(x)\n","        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n","        x = x + self.pos_encoding[tf.newaxis, :length, :]\n","        return x"],"metadata":{"id":"DTr-1ozE2VsR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## 베이스 어텐션"],"metadata":{"id":"OP1HU5xE9cOr"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"CWkHq0kT43UF"},"outputs":[],"source":["class BaseAttention(tf.keras.layers.Layer):\n","    def __init__(self, **kwargs):\n","        super().__init__()\n","        self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)\n","        self.layernorm = tf.keras.layers.LayerNormalization()\n","        self.add = tf.keras.layers.Add()"]},{"cell_type":"markdown","source":["## 크로스 어텐션"],"metadata":{"id":"NTmudkLY9foL"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"zfpv1eC-43UN"},"outputs":[],"source":["class CrossAttention(BaseAttention):\n","    def call(self, x, context):\n","        attn_output, attn_scores = self.mha(\n","                                            query=x,\n","                                            key=context,\n","                                            value=context,\n","                                            return_attention_scores=True)\n","\n","        # 추후에 어텐션 점수 시각화를 위한 변수\n","        self.last_attn_scores = attn_scores\n","\n","        x = self.add([x, attn_output])\n","        x = self.layernorm(x)\n","\n","        return x"]},{"cell_type":"markdown","source":["## 글로벌 셀프 어텐션"],"metadata":{"id":"ubilmFBn9y0z"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"QMIhb4D943UN"},"outputs":[],"source":["class GlobalSelfAttention(BaseAttention):\n","    def call(self, x):\n","        attn_output = self.mha(\n","                              query=x,\n","                              value=x,\n","                              key=x)\n","        x = self.add([x, attn_output])\n","        x = self.layernorm(x)\n","        return x"]},{"cell_type":"markdown","source":["## 코잘 셀프 어텐션"],"metadata":{"id":"5GIFO5kT91Cr"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"0pHTbB7F43UN"},"outputs":[],"source":["class CausalSelfAttention(BaseAttention):\n","    def call(self, x):\n","        attn_output = self.mha(\n","                              query=x,\n","                              value=x,\n","                              key=x,\n","                              use_causal_mask=True)\n","        x = self.add([x, attn_output])\n","        x = self.layernorm(x)\n","        return x"]},{"cell_type":"markdown","source":["## 피드 포워드 레이어"],"metadata":{"id":"B6rhpnrf93ec"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"kpAucJjo43UN"},"outputs":[],"source":["class FeedForward(tf.keras.layers.Layer):\n","    def __init__(self, d_model, dff, dropout_rate=0.1):\n","        super().__init__()\n","        self.seq = tf.keras.Sequential([\n","                                        tf.keras.layers.Dense(units=dff, activation='swish', kernel_initializer=\"he_normal\"),\n","                                        tf.keras.layers.Dense(units=d_model, activation='swish', kernel_initializer=\"he_normal\"),\n","                                        tf.keras.layers.Dropout(rate=dropout_rate)\n","                                      ])\n","        self.add = tf.keras.layers.Add()\n","        self.layer_norm = tf.keras.layers.LayerNormalization()\n","\n","    def call(self, x):\n","        x = self.add([x, self.seq(x)])\n","        x = self.layer_norm(x)\n","        return x"]},{"cell_type":"markdown","source":["## 인코더 레이어"],"metadata":{"id":"V3CvNkiT96rT"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"3Jht-3Vs43UN"},"outputs":[],"source":["class EncoderLayer(tf.keras.layers.Layer):\n","    def __init__(self,*, d_model, num_heads, dff, dropout_rate=0.1):\n","        super().__init__()\n","        self.self_attention = GlobalSelfAttention(\n","                                                  num_heads=num_heads,\n","                                                  key_dim=d_model,\n","                                                  dropout=dropout_rate)\n","        self.ffn = FeedForward(d_model=d_model, dff=dff)\n","\n","    def call(self, x):\n","        x = self.self_attention(x)\n","        x = self.ffn(x)\n","        return x"]},{"cell_type":"markdown","source":["## 인코더"],"metadata":{"id":"v4Mvr39C9_uM"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"kUiJU1Ou43UN"},"outputs":[],"source":["class Encoder(tf.keras.layers.Layer):\n","    def __init__(self, *, num_layers, d_model, num_heads,\n","                dff, vocab_size, dropout_rate=0.1):\n","        super().__init__()\n","        self.d_model = d_model\n","        self.num_layers = num_layers\n","        self.pos_embedding = PositionalEmbedding(vocab_size=vocab_size, d_model=d_model)\n","\n","        self.enc_layers = [\n","                            EncoderLayer(d_model=d_model,\n","                                        num_heads=num_heads,\n","                                        dff=dff,\n","                                        dropout_rate=dropout_rate)\n","                            for _ in range(num_layers)]\n","\n","        self.dropout = tf.keras.layers.Dropout(rate=dropout_rate)\n","\n","    def call(self, x):\n","        # `x` -> token-IDs shape: (batch, seq_len)\n","        x = self.pos_embedding(x)  # Shape `(batch_size, seq_len, d_model)`.\n","        x = self.dropout(x)\n","\n","        for i in range(self.num_layers):\n","            x = self.enc_layers[i](x)\n","\n","        return x  # Shape `(batch_size, seq_len, d_model)`."]},{"cell_type":"markdown","source":["## 디코더 레이어"],"metadata":{"id":"G4u16kV4-BdD"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"Mu_8d4Vh43UO"},"outputs":[],"source":["class DecoderLayer(tf.keras.layers.Layer):\n","    def __init__(self,\n","                *,\n","                d_model,\n","                num_heads,\n","                dff,\n","                dropout_rate=0.1):\n","      super().__init__()\n","      self.causal_self_attention = CausalSelfAttention(\n","                                                      num_heads=num_heads,\n","                                                      key_dim=d_model,\n","                                                      dropout=dropout_rate)\n","      self.cross_attention = CrossAttention(\n","                                            num_heads=num_heads,\n","                                            key_dim=d_model,\n","                                            dropout=dropout_rate)\n","      self.ffn = FeedForward(d_model=d_model, dff=dff)\n","\n","    def call(self, x, context):\n","        x = self.causal_self_attention(x=x)\n","        x = self.cross_attention(x=x, context=context)\n","\n","        # 추후에 어텐션 점수 시각화를 위한 변수\n","        self.last_attn_scores = self.cross_attention.last_attn_scores\n","\n","        x = self.ffn(x)  # Shape `(batch_size, seq_len, d_model)`.\n","        return x"]},{"cell_type":"markdown","source":["## 디코더"],"metadata":{"id":"2vWyJwK9-DLT"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"V7dvc_iu43UO"},"outputs":[],"source":["class Decoder(tf.keras.layers.Layer):\n","    def __init__(self, *, num_layers, d_model, num_heads, dff, vocab_size,\n","                dropout_rate=0.1):\n","        super(Decoder, self).__init__()\n","        self.d_model = d_model\n","        self.num_layers = num_layers\n","        self.pos_embedding = PositionalEmbedding(vocab_size=vocab_size,\n","                                                d_model=d_model)\n","        self.dropout = tf.keras.layers.Dropout(dropout_rate)\n","\n","        self.dec_layers = [\n","                            DecoderLayer(d_model=d_model, num_heads=num_heads,\n","                                        dff=dff, dropout_rate=dropout_rate)\n","                            for _ in range(num_layers)]\n","\n","        self.last_attn_scores = None\n","\n","    def call(self, x, context):\n","        # `x` -> token-IDs shape (batch, target_seq_len)\n","        x = self.pos_embedding(x)  # (batch_size, target_seq_len, d_model)\n","        x = self.dropout(x)\n","\n","        for i in range(self.num_layers):\n","            x  = self.dec_layers[i](x, context)\n","\n","        self.last_attn_scores = self.dec_layers[-1].last_attn_scores\n","\n","        # The shape of x -> (batch_size, target_seq_len, d_model).\n","        return x"]},{"cell_type":"markdown","source":["## 트랜스포머"],"metadata":{"id":"jK2gJDVX-ErL"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"MUa4IA7B43UO"},"outputs":[],"source":["class Transformer(tf.keras.Model):\n","    def __init__(self, *, num_layers, d_model, num_heads, dff,\n","                input_vocab_size, target_vocab_size, dropout_rate=0.1):\n","        super().__init__()\n","        self.encoder = Encoder(num_layers=num_layers, d_model=d_model,\n","                              num_heads=num_heads, dff=dff,\n","                              vocab_size=input_vocab_size,\n","                              dropout_rate=dropout_rate)\n","        self.decoder = Decoder(num_layers=num_layers, d_model=d_model,\n","                              num_heads=num_heads, dff=dff,\n","                              vocab_size=target_vocab_size,\n","                              dropout_rate=dropout_rate)\n","        self.final_layer = tf.keras.layers.Dense(units=target_vocab_size)\n","\n","    def call(self, inputs):\n","        context, x  = inputs\n","        context = self.encoder(context)  # (batch_size, context_len, d_model)\n","        x = self.decoder(x, context)  # (batch_size, target_len, d_model)\n","        logits = self.final_layer(x)  # (batch_size, target_len, target_vocab_size)\n","\n","        try:\n","            # losses/metrics가 커지지 않게 keras_mask를 제거\n","            del logits._keras_mask\n","        except AttributeError:\n","            pass\n","\n","        return logits"]},{"cell_type":"markdown","source":["## 1사이클 스케줄링"],"metadata":{"id":"JF_U_liV-Hqs"}},{"cell_type":"code","source":["class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n","    def __init__(self, d_model, warmup_steps=4000):\n","        super().__init__()\n","        self.d_model = d_model\n","        self.d_model = tf.cast(self.d_model, tf.float32)\n","        self.warmup_steps = warmup_steps\n","\n","    def __call__(self, step):\n","        step = tf.cast(step, dtype=tf.float32)\n","        arg1 = tf.math.rsqrt(step)\n","        arg2 = step * (self.warmup_steps ** -1.5)\n","        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"],"metadata":{"id":"nSG2mZnh52L6"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## 번역기"],"metadata":{"id":"rS2UUDNg-MNL"}},{"cell_type":"code","source":["class Translator(tf.Module):\n","    def __init__(self, tokenizers, transformer):\n","        self.tokenizers = tokenizers\n","        self.transformer = transformer\n","\n","    def __call__(self, sentence, max_length):\n","        # input sentence는 포르투칼어로 [START]와 [END]토큰을 추가\n","        assert isinstance(sentence, tf.Tensor)\n","        if len(sentence.shape) == 0:\n","            sentence = sentence[tf.newaxis]\n","\n","        sentence = self.tokenizers.pt.tokenize(sentence).to_tensor()\n","        encoder_input = sentence\n","\n","        # 출력 언어는 영어이기 때문에 [START] 토큰으로 시작\n","        start_end = self.tokenizers.en.tokenize([''])[0]\n","        start = start_end[0][tf.newaxis]\n","        end = start_end[1][tf.newaxis]\n","\n","        # 파이썬 리스트 대신에 tf.TensorArray를 사용\n","        # 이를 통해 tf.function으로 dynamic-loop 추적 가능\n","        output_array = tf.TensorArray(dtype=tf.int64, size=0, dynamic_size=True)\n","        output_array = output_array.write(0, start)\n","\n","        for i in tf.range(max_length):\n","            output = tf.transpose(output_array.stack())\n","            predictions = self.transformer([encoder_input, output], training=False)\n","\n","            # `seq_len` 차원에서 마지막 토큰을 선택\n","            predictions = predictions[:, -1:, :]  # Shape `(batch_size, 1, vocab_size)`.\n","            predicted_id = tf.argmax(predictions, axis=-1)\n","\n","            output_array = output_array.write(i + 1, predicted_id[0])\n","\n","            if predicted_id == end:\n","                break\n","\n","        output = tf.transpose(output_array.stack())\n","        # output shape -> `(1, tokens)`.\n","        text = tokenizers.en.detokenize(output)[0]  # Shape: `()`.\n","\n","        tokens = tokenizers.en.lookup(output)[0]\n","\n","        # tf.function으로 루프의 마지막 반복에서 계산된 attention_weights가 사용되지 않게 예방\n","        # 이를 통해 루프 외부에서 재계산\n","        self.transformer([encoder_input, output[:,:-1]], training=False)\n","        attention_weights = self.transformer.decoder.last_attn_scores\n","\n","        return text, tokens, attention_weights"],"metadata":{"id":"AXQGW8hN6ims"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## 확장 번역기"],"metadata":{"id":"2y0EmVpO-UKT"}},{"cell_type":"code","source":["class ExportTranslator(tf.Module):\n","    def __init__(self, translator):\n","        self.translator = translator\n","\n","    @tf.function(input_signature=[tf.TensorSpec(shape=[], dtype=tf.string)])\n","    def __call__(self, sentence):\n","        (result,\n","        tokens,\n","        attention_weights) = self.translator(sentence, max_length=MAX_TOKENS)\n","\n","        return result"],"metadata":{"id":"LWc8Hxd17S3x"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# 함수"],"metadata":{"id":"BwmcAcOU1gHK"}},{"cell_type":"code","source":["def prepare_tokenizer(tokenizers, MAX_TOKENS):\n","    def prepare_batch(pt, en):\n","        pt = tokenizers.pt.tokenize(pt)      # Ragged Tensor가 출력\n","        pt = pt[:, :MAX_TOKENS]    # MAX_TOKENS으로 자르기\n","        pt = pt.to_tensor()  # 제로 패딩한 텐서로 전환\n","\n","        en = tokenizers.en.tokenize(en)\n","        en = en[:, :(MAX_TOKENS + 1)]\n","        en_inputs = en[:, :-1].to_tensor()  # [END] 토큰 제거\n","        en_labels = en[:, 1:].to_tensor()   # [START] 토큰 제거\n","\n","        return (pt, en_inputs), en_labels\n","    return prepare_batch"],"metadata":{"id":"f7PL2Ezd1u0x"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# tf.data.Dataset을 배치하는 함수\n","def make_batches(ds: tf.data.Dataset, tokenizers: tf.Module, MAX_TOKENS: int, BUFFER_SIZE: int, BATCH_SIZE: int):\n","    \"\"\"tf.data.Dataset을 배치 크기만큼 적용하는 함수\n","\n","    Args:\n","        ds: tf.data.Dataset\n","        tokenizers: tfds로 불러온 토크나이저 -> tf.Module\n","        MAX_TOEKNS: 최대 토큰 길이 -> int\n","        BUFFER_SIZE: 버퍼 크기 -> int\n","        BATCH_SIZE: 배치 크기 -> int\n","\n","    Return\n","        tf.data.Dataset\n","    \"\"\"\n","    ds = ds.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n","    ds = ds.map(prepare_tokenizer(tokenizers=tokenizers, MAX_TOKENS=MAX_TOKENS), tf.data.AUTOTUNE).prefetch(buffer_size=tf.data.AUTOTUNE)\n","    return ds"],"metadata":{"id":"ExvA9ILr1vZD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# 마스킹된 손실함수\n","def masked_loss(label: tf.Tensor, pred: tf.Tensor) -> float:\n","    \"\"\"마스킹을 포함한 예측값과 실제값의 손실값을 계산하는 함수\n","\n","    Args:\n","        label: 실제값 -> tf.Tensor\n","        pred: 예측값 -> tf.Tensor\n","\n","    Returns:\n","        손실값 -> float\n","    \"\"\"\n","    mask = label != 0\n","    loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n","    loss = loss_object(y_true=label, y_pred=pred)\n","\n","    mask = tf.cast(mask, dtype=loss.dtype)\n","    loss *= mask\n","\n","    loss = tf.reduce_sum(input_tensor=loss) / tf.reduce_sum(input_tensor=mask)\n","    return loss\n","\n","# 마스킹된 정확도 함수\n","def masked_accuracy(label: tf.Tensor, pred: tf.Tensor) -> float:\n","    \"\"\"마스킹을 포함한 예측값과 실제값의 정확도를 계산하는 함수\n","\n","    Args:\n","        label: 실제값 -> tf.Tensor\n","        pred: 예측값 -> tf.Tensor\n","\n","    Returns:\n","        정확도 -> float\n","    \"\"\"\n","    pred = tf.argmax(input=pred, axis=2)\n","    label = tf.cast(label, dtype=pred.dtype)\n","    matched = label == pred\n","\n","    mask = label != 0\n","\n","    matched = matched & mask\n","\n","    matched = tf.cast(matched, dtype=tf.float32)\n","    mask = tf.cast(mask, dtype=tf.float32)\n","    return tf.reduce_sum(input_tensor=matched) / tf.reduce_sum(input_tensor=mask)"],"metadata":{"id":"-JcoGqwl6IWS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# 번역 비교 함수\n","def print_translation(sentence: str, tokens: tf.Tensor, ground_truth: str) -> None:\n","    \"\"\"입력 텍스트 문자열과 예측 텍스트 문자열, 실제 텍스트 문자열을 출력 비교하는 함수\n","\n","    Args:\n","        sentence: 입력 텍스트 문자열 -> str\n","        tokens: 예측 인코딩 텐서 -> tf.Tensor\n","        ground_truth: 실제 비교 텍스트 문자열 -> str\n","    \"\"\"\n","    print(f'{\"Input:\":15s}: {sentence}')\n","    print(f'{\"Prediction\":15s}: {tokens.numpy().decode(\"utf-8\")}')\n","    print(f'{\"Ground truth\":15s}: {ground_truth}')"],"metadata":{"id":"4xj0kwnc7oa5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# 변수"],"metadata":{"id":"G9dSTCfG1hEh"}},{"cell_type":"code","source":["SEED = 1234\n","tf.random.set_seed(SEED)\n","\n","MAX_TOKENS = 128\n","\n","BUFFER_SIZE = 20000\n","BATCH_SIZE = 64"],"metadata":{"id":"3N14dZWf15Gh"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# 데이터"],"metadata":{"id":"DlpRxU2w15zp"}},{"cell_type":"markdown","source":["## 데이터 다운로드"],"metadata":{"id":"utgbzBmHHTFn"}},{"cell_type":"markdown","source":["이 데이터는 52,000개의 학습 데이터와 1,800개의 검증 데이터 및 1,200개의 테스트 데이터로 구성"],"metadata":{"id":"q8h81pfNHYe2"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"8q9t4FmN96eN","colab":{"base_uri":"https://localhost:8080/","height":170,"referenced_widgets":["43276f2848d5450087b06a52e5fcb52e","5919574ff7524544a16b98ef60929ff5","c3e74b1904134ddca349d03fe087b022","4894798e1560439a9768d5dbe5af494d","bfecdd93bcae458ea3515cdcd2285bfc","b0522642574843ecbc8d13415bf7d876","f0a51618cfec4e1b94128f89be9bfc01","566c72dda8c24aa686e4deea5b684676","8e496d04f0fc47eb94ecfd4eedb04249","5f8686fad8934440b97233b9db738a28","46761aa64c604e848373084e7a9cac9d","516237a3dfa444f1bcd313edc81fbc44","94cc956b897f444ea68820bbbe664dac","32f0068103bc40f7b153655ae0f698ca","117a45652f90414581aed1844128b68b","8778da9ef3714955a65a9d7347ff7d6e","fda55d7e78c94fe9877e19938e59fc42","d45345545cfb4fda8b5ea9222e6960d3","5a9d0b6ac2c84ffea2b503c718f8de06","7538303a85194ee29d0cd5c8234da6db","522118041cfe4182a615d4cc9725d349","27c47044363d47fbb447da999a593926","8d956d9b37644d12bc6b0abafecbd7d6","cecb63ff967d45669e94c206bb2d4076","96deec1151fb4ffe81db1858f1058762","2daa9bd0d6d3478b95c87e5afa6496db","2e3bfbfa17ac4145a1e50903bccc5308","c421a608e02b46beae73afd95a7430c8","234dc51befd74ba9a7af1b820d6074f5","49f96943370041a7be991adbecb64e97","7694867b94394a67b8f0dbcfa08bcff0","e3fae78dbe664d2f8c51a10f56248a7d","d88a837f69854dcf88d711865cf9cc97","10a00c5e760e4b23a81f19b78bcb8163","727fbc26e9dd4a81b39ed4d796d4bd95","320ebfe94e264f5194881de168f8c27b","3215b96a369a418ba0762d3bec83c88d","8e656799220b463aa1e4e1cef8bec17e","d01cb942e8664f139524fa42737d7858","f45904c00d4046adab624800596aa153","ae2ae93367144d02b49dd8d9e1f0118b","d0b0096959de4dfeb695200974b0e622","3121ffd5d95b4866b76436a5fe47a270","211196069e224e3eaddcf8cfb7f06601","69758a2e414944c3b00765b7f433a95d","ae620515eaef4ee1962b7f73e4bfcb76","433a1d3ca73a42ccbbb8670df14cc70c","92162094ca7f408692e097f40c713b77","0f428af76f234a99871277ea2964c5a5","0f17fc9272134131a4747a964892fc89","5d2bf0f5b6e64e5bbafb9a42e777bbf2","c39e2afb29a84d8bb197c1aea0b499ef","6bad5f8039094968b79a26cb5ae23288","7fee9841b9c8417bbb79fa7236809748","35dd13bfd6e34022abaee971ceae47ac","9031896bd7d443e7b98d39a3351d2566","562ab14e85994040b37ce863c01bfb7d","d2095fdb584a4ad7b813b69ae499e192","c7287e704cad437c9b1f52902c6f6147","a51ad239145641ae9912d5784b70d396","211cfb647fc34fb990f9a10001367f4c","8f5e30185e7a43689cca17d26136d325","48eab2762fdb4c69b5e49a9546103a1f","d35ca563c8b64c368b64317ca9830421","6937dd0896a44933ba1a63f6988af596","ebb1468294a14e4384d7921a9ec95429","55cc534b52f9453fa6880f9259f97179","fc1bc2e98e5e42358fef5b5e22318901","0cade649c70e4985915c1cc32db481bb","62e34718a06649ffaad33074cc4d5bd3","b343cb3a319a4aed82ad123a151b29fa","998c375c01e54029b899090d6847cb9b","6014658bba0748869115a16b84760f6a","a3918bbef7704ccdb65c546cd28b277c","303889f13e7b40aaa97fc36166b794da","b1aa8e164d2c46f19b1677b6df85b820","9bd4959bf13748bd818acfb947b2cfce","1fdd342c87514b52ae1715e7e60c60e1","5250c7c093f44e41a4db605af76e6115","e27420d9d3e3489294ea575f837f1134","8f823403ebc0424e9c7f234f4331dcf1","4b8a60042ea845879b30b0a17e404620","fde787f1305048c696a9c2fd39a9a6b1","84eb98c3b4a6443bb929faac041fa210","d4062553fc6547488563451eddeb9cb7","95632fd4607e4c14878abc11c432144e","686682c4d9a24f94ba19fa1838177251","803673935a0c4f5abb5246c0c5cd3904","60a010ec3e7a4541972df5419ed1961e","e2f0e147902340cda73996119ef9de8f","b02090a0272b49d19502aea2ff72deea","24a66ce200404ac6ac5353c1c503b299","38b8b6856b854a9e8adfa687dd86941e","5498751a235b441b989b5a6c4f4127e9","2f11e674139442ea884d26aebf43af4a","80bc51204710477394cd0049ef2dc2b3","5da90bd31cec4c49bade659a725d1480","111648081f694975afaef411933e99b0","d3f4d62262244237bf052cdb370f0d67","f9ef65e4de9d4c8987a6fe9f4f93188a","41094d8baa5242ebb4f89955f9530616","7be7d795450f4e17a2d5b6838df17f34","9463853cb5d7485fa939fc79b0c385fb","b9a3b93a76d646b7b2cb05af5160da94","0a7fc203034343faa4486a7a0a682746","83b73ede65f440aaa35a526498def1ce","e82c7d973eb443e3bbea25be19e873db","c0731bc2210d413f848a7dadf8373255","25402a23331042b3acd1a9861509e0d1","ca10d72db12f4e9b9ef42efedae2a94c"]},"executionInfo":{"status":"ok","timestamp":1721906591382,"user_tz":-540,"elapsed":29842,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"7574a47b-77b8-41bb-accf-f888e527663f"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading and preparing dataset 124.94 MiB (download: 124.94 MiB, generated: Unknown size, total: 124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\n"]},{"output_type":"display_data","data":{"text/plain":["Dl Completed...: 0 url [00:00, ? url/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"43276f2848d5450087b06a52e5fcb52e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Dl Size...: 0 MiB [00:00, ? MiB/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"516237a3dfa444f1bcd313edc81fbc44"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Extraction completed...: 0 file [00:00, ? file/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8d956d9b37644d12bc6b0abafecbd7d6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating splits...:   0%|          | 0/3 [00:00<?, ? splits/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"10a00c5e760e4b23a81f19b78bcb8163"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating train examples...:   0%|          | 0/51785 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"69758a2e414944c3b00765b7f433a95d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-tra…"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9031896bd7d443e7b98d39a3351d2566"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating validation examples...:   0%|          | 0/1193 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"55cc534b52f9453fa6880f9259f97179"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-val…"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1fdd342c87514b52ae1715e7e60c60e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating test examples...:   0%|          | 0/1803 [00:00<?, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"60a010ec3e7a4541972df5419ed1961e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-tes…"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9ef65e4de9d4c8987a6fe9f4f93188a"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Dataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\n"]}],"source":["examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True, as_supervised=True)\n","train_examples, val_examples = examples['train'], examples['validation']"]},{"cell_type":"markdown","source":["## 데이터 토크나이저 다운로드"],"metadata":{"id":"cbfXkKbuHt_X"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"QToMl0NanZPr","colab":{"base_uri":"https://localhost:8080/","height":72},"executionInfo":{"status":"ok","timestamp":1721906591383,"user_tz":-540,"elapsed":24,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"606fb645-316d-4f15-a867-e3e896c2e697"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading data from https://storage.googleapis.com/download.tensorflow.org/models/ted_hrlr_translate_pt_en_converter.zip\n","\u001b[1m184801/184801\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\n"]},{"output_type":"execute_result","data":{"text/plain":["'./ted_hrlr_translate_pt_en_converter.zip'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":23}],"source":["model_name = 'ted_hrlr_translate_pt_en_converter'\n","tf.keras.utils.get_file(\n","                        f'{model_name}.zip',\n","                        f'https://storage.googleapis.com/download.tensorflow.org/models/{model_name}.zip',\n","                        cache_dir='.', cache_subdir='', extract=True\n","                        )"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"h5dbGnPXnuI1"},"outputs":[],"source":["tokenizers = tf.saved_model.load(model_name)"]},{"cell_type":"markdown","source":["## 토큰화 길이 시각화"],"metadata":{"id":"zUpBtJwPH16w"}},{"cell_type":"code","source":["lengths = []\n","\n","for pt_examples, en_examples in train_examples.batch(1024):\n","    pt_tokens = tokenizers.pt.tokenize(pt_examples)\n","    lengths.append(pt_tokens.row_lengths())\n","\n","    en_tokens = tokenizers.en.tokenize(en_examples)\n","    lengths.append(en_tokens.row_lengths())"],"metadata":{"id":"rfjoJSRvG9AQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["all_lengths = np.concatenate(lengths)\n","\n","plt.hist(all_lengths, np.linspace(0, 500, 101))\n","plt.ylim(plt.ylim())\n","max_length = max(all_lengths)\n","plt.plot([max_length, max_length], plt.ylim())\n","plt.title(f'Maximum tokens per example: {max_length}')\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":452},"id":"LkrsX2ZfHEOZ","executionInfo":{"status":"ok","timestamp":1721906695585,"user_tz":-540,"elapsed":547,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"b94d26cd-62f9-41f7-f9fc-3678b13431c1"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAjkAAAGzCAYAAADNKAZOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5A0lEQVR4nO3deVyVZf7/8fdBPIALB00BGVE097VCQ1yykhGVLBtz0qxxS9PBJrWvpU3j0jRpi5qFaU6TzlROalNarhEqZu4k45I6VjpaimuCkoLC9fvDH/d4BLcCgavX8/E4j8c51/257/u6L2477+7tuIwxRgAAAJbxKe4OAAAAFAVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOcAUul0vjxo0r7m6UKH379lWFChWKuxsoYfi3gpKIkIMSZ/bs2XK5XHK5XFqzZk2+6cYYhYeHy+Vy6Z577imGHpZeBw8e1Lhx45SamlrcXQGu25kzZzRgwAA1adJEHo9HFSpUUPPmzTV16lSdO3fOqzYpKUn9+/dXvXr1VK5cOdWuXVuPPvqoDh06VOCy165dq7Zt26pcuXIKDQ3VH/7wB50+ffpGbBaKkG9xdwC4HH9/f82ZM0dt27b1ak9OTtZ3330nPz+/Iu/DmTNn5Otrzz+TgwcPavz48YqIiNAtt9xS3N0BrsuZM2e0Y8cOdenSRREREfLx8dHatWs1fPhwbdiwQXPmzHFqn376aZ04cUI9evRQ3bp19e233yohIUGLFi1SamqqQkNDndrU1FR16NBBDRs21OTJk/Xdd9/plVde0Z49e7R06dLi2FQUEnv+6w3rdOnSRfPnz9drr73mFTTmzJmjyMhIHTt2rMj74O/vX+TrQOmXm5ur7Oxs9pciVrlyZa1fv96rbfDgwfJ4PEpISNDkyZOd8DJ58mS1bdtWPj7/O2HRqVMntW/fXgkJCXr++eed9meeeUaVKlXSqlWrFBgYKEmKiIjQwIED9emnn6pjx443YOtQFDhdhRKrV69eOn78uBITE5227OxsffDBB3rooYcKnOeVV15R69atddNNNykgIECRkZH64IMPvGpmzZoll8ult99+26v9hRdekMvl0pIlS5y2S68zGDdunFwul/7zn//o4YcflsfjUdWqVfWnP/1JxhgdOHBA9913nwIDAxUaGqpJkyZ5rSPvVNy+ffu82letWiWXy6VVq1Y5bXfeeaeaNGmirVu3qn379ipXrpzq1KnjbE9ycrKioqIUEBCg+vXr67PPPrvieK5atUotW7aUJPXr1885JTh79mynZv78+YqMjFRAQICqVKmihx9+WN9///0Vlytd+D/hqlWr6s4773QO8X///ffq37+/QkJC5Ofnp8aNG+cb87ztnjdvnv7yl7+oevXq8vf3V4cOHfT111971e7Zs0fdu3dXaGio/P39Vb16dfXs2VPp6elX7FveOKakpKh169YKCAhQrVq1NGPGjHy1WVlZGjt2rOrUqSM/Pz+Fh4frqaeeUlZWlledy+XS0KFD9d5776lx48by8/PTsmXLrtiPpUuXql27dipfvrwqVqyouLg47dixw5m+YsUK+fj4aMyYMV7zzZkzRy6XS9OnT3faZs2apbvvvlvBwcHy8/NTo0aNvKbniYiI0D333KNVq1apRYsWCggIUNOmTZ397MMPP1TTpk3l7++vyMhIbdmyxWv+vOuvvv32W8XGxqp8+fIKCwvTc889J2PMFbdXurZ9QJL279+vXbt2XXV5lxMRESFJOnnypNN2xx13eAWcvLbKlStr586dTltGRoYSExP18MMPOwFHkn73u9+pQoUKmjdv3k/uF0oAA5Qws2bNMpLMpk2bTOvWrc0jjzziTFuwYIHx8fEx33//valZs6aJi4vzmrd69erm97//vUlISDCTJ082t99+u5FkFi1a5FV3zz33GI/HY/bv32+MMWbr1q3G7XabAQMGeNVJMmPHjnU+jx071kgyt9xyi+nVq5d54403TFxcnJFkJk+ebOrXr2+GDBli3njjDdOmTRsjySQnJ+fbtr1793qtZ+XKlUaSWblypdPWvn17ExYWZsLDw83IkSPN66+/bho1amTKlClj3n//fRMaGmrGjRtnXn31VfOrX/3KeDwek5GRcdlxTUtLM88995yRZAYNGmTeeecd884775hvvvnGq28tW7Y0U6ZMMaNGjTIBAQEmIiLC/PDDD85y+vTpY8qXL+983rhxo6lUqZL59a9/bX788UdnXdWrVzfh4eHmueeeM9OnTzf33nuvkWSmTJmSb7tvvfVWExkZaaZMmWLGjRtnypUrZ26//XanLisry9SqVcuEhYWZ559/3rz11ltm/PjxpmXLlmbfvn2X3eaLxzE4ONgMHTrUvPbaa6Zt27ZGkvnb3/7m1OXk5JiOHTuacuXKmWHDhpk333zTDB061Pj6+pr77rvPa5mSTMOGDU3VqlXN+PHjzbRp08yWLVsu24d//OMfxuVymU6dOpnXX3/dvPjiiyYiIsIEBQV57Qvx8fHG19fXpKSkGGOMOXjwoKlcubKJiYkxubm5Tl3Lli1N3759zZQpU8zrr79uOnbsaCSZhIQEr/XWrFnT1K9f31SrVs2MGzfOTJkyxfzqV78yFSpUMO+++66pUaOGmThxopk4caLxeDymTp06Jicnx5m/T58+xt/f39StW9c88sgjJiEhwdxzzz1GkvnTn/6Ub0wu/rdyrftA3t/oer6OsrKyzNGjR83+/fvNhx9+aEJDQ03NmjXNuXPnrjjfqVOnjNvtNoMGDXLa1qxZYySZuXPn5qtv27atue222665Xyh5CDkocS4OOQkJCaZixYrOl2ePHj3MXXfdZYwxBYacvLo82dnZpkmTJubuu+/2aj906JCpXLmy+fWvf22ysrLMrbfeamrUqGHS09O96i4Xci7+j+T58+dN9erVjcvlMhMnTnTaf/jhBxMQEGD69OmTb9uuNeRIMnPmzHHadu3aZSQZHx8fs379eqd9+fLlRpKZNWuWuZJNmzYVWJednW2Cg4NNkyZNzJkzZ5z2RYsWGUlmzJgxTtvFIWfNmjUmMDDQxMXFmbNnzzo1AwYMMNWqVTPHjh3zWk/Pnj2Nx+Nx/k55292wYUOTlZXl1E2dOtVIMtu2bTPGGLNlyxYjycyfP/+K21eQvHGcNGmS05aVlWVuueUWExwcbLKzs40xxrzzzjvGx8fHfP75517zz5gxw0gyX3zxhdOW9zfYsWPHVdd/6tQpExQUZAYOHOjVnpaWZjwej1d7ZmamqVOnjmncuLE5e/asiYuLM4GBgea///2v17yX7ufGGBMbG2tq167t1VazZk0jyaxdu9Zpy9tXAgICvJb75ptv5tsH+/TpYySZxx9/3GnLzc01cXFxxu12m6NHj3qNycX/Vq51HzDm+kPOP//5TyPJebVo0cJs3br1qvP9+c9/NpJMUlKS0zZ//nwjyaxevTpffY8ePUxoaOg19wslD6erUKL99re/1ZkzZ7Ro0SKdOnVKixYtuuypKkkKCAhw3v/www9KT09Xu3bt9OWXX3rVhYaGatq0aUpMTFS7du2Umpqqt99+2+tw9ZU8+uijzvsyZcqoRYsWMsZowIABTntQUJDq16+vb7/99lo3N58KFSqoZ8+ezuf69esrKChIDRs2VFRUlNOe9/6nrmvz5s06cuSIfv/733tdVxIXF6cGDRpo8eLF+eZZuXKlYmNj1aFDB3344YfOheDGGP3rX/9S165dZYzRsWPHnFdsbKzS09Pz/T369esnt9vtfG7Xrp3X9ng8HknS8uXL9eOPP1739vn6+uqxxx5zPrvdbj322GM6cuSIUlJSJF04VdewYUM1aNDAq8933323s70Xa9++vRo1anTVdScmJurkyZPq1auX13LLlCmjqKgor+WWK1dOs2fP1s6dO3XHHXdo8eLFmjJlimrUqOG1zIv38/T0dB07dkzt27fXt99+m+/0XaNGjRQdHe18zttX7r77bq/lXmkfGjp0qPM+71Rddnb2ZU+RXu8+sGrVqms6/ZXnrrvuUmJioubPn6/BgwerbNmyyszMvOI8q1ev1vjx4/Xb3/7W+ZtKFy5mllTgjQz+/v7OdJROXHiMEq1q1aqKiYnRnDlz9OOPPyonJ0cPPPDAZesXLVqk559/XqmpqV7XUbhcrny1PXv21LvvvqvFixdr0KBB6tChwzX369IvHY/HI39/f1WpUiVf+/Hjx695uZeqXr16vr57PB6Fh4fna5MuBLuf4r///a+kCyHqUg0aNMh3K//Zs2cVFxenyMhIzZs3z+vC8KNHj+rkyZOaOXOmZs6cWeD6jhw54vX50vGsVKmSpP9tT61atTRixAhNnjxZ7733ntq1a6d7773XuS7qasLCwlS+fHmvtnr16kmS9u3bp1atWmnPnj3auXOnqlatek19rlWr1lXXK124lkiS1xfrxS4N1m3atNGQIUM0bdo0xcbGqn///vnm+eKLLzR27FitW7cuX+hLT0/3GpOC9lVJ17wP+fj4qHbt2l5tF49dQX7KPnA9QkJCFBISIkl64IEH9MILL+jXv/619uzZ43XXVJ5du3bp/vvvV5MmTfTWW295TcsLjJdedyVd2M8vDpQofQg5KPEeeughDRw4UGlpaercubOCgoIKrPv8889177336o477tAbb7yhatWqqWzZspo1a5bXraV5jh8/rs2bN0uSvvrqK+Xm5ua7UPFyypQpc01tkrz+D7WgsCVJOTk517yea11XUfLz81OXLl20cOFCLVu2zOt5Rbm5uZKkhx9+WH369Clw/mbNmnl9vpbtmTRpkvr27auFCxfq008/1R/+8AdNmDBB69evV/Xq1X/uJik3N1dNmzbV5MmTC5x+aSi41i+/vPF45513CvwCvvQRBVlZWc6Fwd98841+/PFHlStXzpn+zTffqEOHDmrQoIEmT56s8PBwud1uLVmyRFOmTHHWl6c49qGfsg/8HA888ID++Mc/auHChV5H7CTpwIED6tixozwej5YsWaKKFSt6Ta9WrZokFfj8nEOHDiksLKzQ+okbj5CDEu/+++/XY489pvXr12vu3LmXrfvXv/4lf39/LV++3OvQ86xZswqsj4+P16lTpzRhwgSNHj1ar776qkaMGFHo/b9Y3hGKi+8Ckf53JKWoXS5k1axZU5K0e/fufEccdu/e7Uy/eDnvvfee7rvvPvXo0UNLly7VnXfeKenC0beKFSsqJydHMTExhdr/pk2bqmnTpnr22We1du1atWnTRjNmzPC6HbggBw8eVGZmptfRnP/85z+S/ndnzs0336x///vf6tChw2XH6ae4+eabJUnBwcHXNB5jx47Vzp079corr+jpp5/WqFGj9NprrznTP/nkE2VlZenjjz/2Okpz6em0wpKbm6tvv/3WOXoj5R+7SxXlPlCQvFNKl56qO378uDp27KisrCwlJSU5geZiTZo0ka+vrzZv3qzf/va3Tnt2drZSU1O92lD6cE0OSrwKFSpo+vTpGjdunLp27XrZujJlysjlcnkdFdm3b58WLFiQr/aDDz7Q3LlzNXHiRI0aNUo9e/bUs88+6/zHu6jkfeGtXr3aacvJybnsIf3Clvclf2nIatGihYKDgzVjxgyvw/ZLly7Vzp07FRcXl29ZbrdbH374oVq2bKmuXbtq48aNki78Hbp3765//etf2r59e775jh49et39zsjI0Pnz573amjZtKh8fnwJPM1zq/PnzevPNN53P2dnZevPNN1W1alVFRkZKunD91/fff6+//vWv+eY/c+bMVa/5uJzY2FgFBgbqhRdeyPdUXsl7PDZs2KBXXnlFw4YN05NPPqmRI0cqISFBycnJTk3eEZiLj7ikp6dfNswXhoSEBOe9MUYJCQkqW7bsZU/xXu8+cK23kB87dqzAI015p6BatGjhtGVmZqpLly76/vvvtWTJEtWtW7fAZXo8HsXExOjdd9/VqVOnnPZ33nlHp0+fVo8ePa7aL5RcHMlBqXC5Q94Xi4uL0+TJk9WpUyc99NBDOnLkiKZNm6Y6depo69atTt2RI0c0ZMgQ3XXXXc4FlQkJCVq5cqX69u2rNWvWXPNpq+vVuHFjtWrVSqNHj9aJEydUuXJlvf/++/m+wIvKzTffrKCgIM2YMUMVK1ZU+fLlFRUVpVq1aunFF19Uv3791L59e/Xq1UuHDx/W1KlTFRERoeHDhxe4vICAAC1atEh33323OnfurOTkZDVp0kQTJ07UypUrFRUVpYEDB6pRo0Y6ceKEvvzyS3322Wc6ceLEdfV7xYoVGjp0qHr06KF69erp/Pnzeuedd5wv06sJCwvTiy++qH379qlevXqaO3euUlNTNXPmTJUtW1aS9Mgjj2jevHkaPHiwVq5cqTZt2ignJ0e7du3SvHnztHz5cq8v0WsVGBio6dOn65FHHtFtt92mnj17qmrVqtq/f78WL16sNm3aKCEhQWfPnlWfPn1Ut25d/eUvf5EkjR8/Xp988on69eunbdu2qXz58urYsaPcbre6du2qxx57TKdPn9Zf//pXBQcHX/YnC34Of39/LVu2TH369FFUVJSWLl2qxYsX65lnnrns9UuSrmsf+N3vfqfk5OSrnip79913NWPGDHXr1k21a9fWqVOntHz5ciUmJqpr165eRyF79+6tjRs3qn///tq5c6fXs3EqVKigbt26OZ//8pe/qHXr1mrfvr0GDRqk7777TpMmTVLHjh3VqVOnnzBqKDGK45Yu4EouvoX8Sgq6hfxvf/ubqVu3rvHz8zMNGjQws2bNcm77zvOb3/zGVKxYMd/zVRYuXGgkmRdffNFp02VuIb/41llj8j87Jk/79u1N48aNvdq++eYbExMTY/z8/ExISIh55plnTGJiYoG3kF867+W2O6+v8fHx+dovtXDhQtOoUSPj6+ub73byuXPnmltvvdX4+fmZypUrm969e5vvvvvuqtt67Ngx06hRIxMaGmr27NljjDHm8OHDJj4+3oSHh5uyZcua0NBQ06FDBzNz5kxnvrxbyC+9NXzv3r1effv2229N//79zc0332z8/f1N5cqVzV133WU+++yzq25v3jhu3rzZREdHG39/f1OzZs18z5Qx5sKt9C+++KJp3Lix8fPzM5UqVTKRkZFm/PjxXo8XuNaxvtjKlStNbGys8Xg8xt/f39x8882mb9++ZvPmzcYYY4YPH27KlCljNmzY4DXf5s2bja+vrxkyZIjT9vHHH5tmzZoZf39/ExERYV588UXz9ttv53s8wfXsK3lj/vLLLztteX/rb775xnmGUEhIiBk7dqzX83TylnnxvxVjrm0fMObabyHftGmT6dGjh6lRo4bx8/Mz5cuXN7fddpuZPHlyvmfk5N0+X9CrZs2a+Zb9+eefm9atWxt/f39TtWpVEx8ff8XnTqF0cBlzg65UBIBicOedd+rYsWMFnjbBlfXt21cffPABP1SJUotrcgAAgJUIOQAAwEqEHAAAYCWuyQEAAFbiSA4AALASIQcAAFjpF/0wwNzcXB08eFAVK1Ys1Me4AwCAomOM0alTpxQWFnbFh7f+okPOwYMH8/3oHgAAKB0OHDhwxR/o/UWHnLxfoz1w4IACAwOLuTcAUAiyM6VJ9S+8f3K35C5/5XqgFMrIyFB4eHi+X5W/1C865OSdogoMDCTkALBDdhnJ7/+ffg8MJOTAale71IQLjwEAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACs5FvcHfglixi1OF/bvolxxdATAADsw5EcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYKXrCjkTJkxQy5YtVbFiRQUHB6tbt27avXu3V83Zs2cVHx+vm266SRUqVFD37t11+PBhr5r9+/crLi5O5cqVU3BwsEaOHKnz58971axatUq33Xab/Pz8VKdOHc2ePTtff6ZNm6aIiAj5+/srKipKGzduvJ7NAQAAFruukJOcnKz4+HitX79eiYmJOnfunDp27KjMzEynZvjw4frkk080f/58JScn6+DBg/rNb37jTM/JyVFcXJyys7O1du1a/f3vf9fs2bM1ZswYp2bv3r2Ki4vTXXfdpdTUVA0bNkyPPvqoli9f7tTMnTtXI0aM0NixY/Xll1+qefPmio2N1ZEjR37OeAAAAEu4jDHmp8589OhRBQcHKzk5WXfccYfS09NVtWpVzZkzRw888IAkadeuXWrYsKHWrVunVq1aaenSpbrnnnt08OBBhYSESJJmzJihp59+WkePHpXb7dbTTz+txYsXa/v27c66evbsqZMnT2rZsmWSpKioKLVs2VIJCQmSpNzcXIWHh+vxxx/XqFGjCuxvVlaWsrKynM8ZGRkKDw9Xenq6AgMDf+ow/GQRoxbna9s3Me6G9wOARbIzpRfCLrx/5qDkLl+8/QGKQEZGhjwez1W/v3/WNTnp6emSpMqVK0uSUlJSdO7cOcXExDg1DRo0UI0aNbRu3TpJ0rp169S0aVMn4EhSbGysMjIytGPHDqfm4mXk1eQtIzs7WykpKV41Pj4+iomJcWoKMmHCBHk8HucVHh7+czYfAACUYD855OTm5mrYsGFq06aNmjRpIklKS0uT2+1WUFCQV21ISIjS0tKcmosDTt70vGlXqsnIyNCZM2d07Ngx5eTkFFiTt4yCjB49Wunp6c7rwIED17/hAACgVPD9qTPGx8dr+/btWrNmTWH2p0j5+fnJz8+vuLsBAABugJ90JGfo0KFatGiRVq5cqerVqzvtoaGhys7O1smTJ73qDx8+rNDQUKfm0rut8j5frSYwMFABAQGqUqWKypQpU2BN3jIAAMAv23WFHGOMhg4dqo8++kgrVqxQrVq1vKZHRkaqbNmySkpKctp2796t/fv3Kzo6WpIUHR2tbdu2ed0FlZiYqMDAQDVq1MipuXgZeTV5y3C73YqMjPSqyc3NVVJSklMDAAB+2a7rdFV8fLzmzJmjhQsXqmLFis71Lx6PRwEBAfJ4PBowYIBGjBihypUrKzAwUI8//riio6PVqlUrSVLHjh3VqFEjPfLII3rppZeUlpamZ599VvHx8c6ppMGDByshIUFPPfWU+vfvrxUrVmjevHlavPh/dyONGDFCffr0UYsWLXT77bfr1VdfVWZmpvr161dYYwMAAEqx6wo506dPlyTdeeedXu2zZs1S3759JUlTpkyRj4+PunfvrqysLMXGxuqNN95wasuUKaNFixZpyJAhio6OVvny5dWnTx8999xzTk2tWrW0ePFiDR8+XFOnTlX16tX11ltvKTY21ql58MEHdfToUY0ZM0ZpaWm65ZZbtGzZsnwXIwMAgF+mn/WcnNLuWu+zLyo8JwdAoeM5OfgFuCHPyQEAACipCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKz0k3+7CkXj0tvKuaUcAICfhiM5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGCl6w45q1evVteuXRUWFiaXy6UFCxZ4Te/bt69cLpfXq1OnTl41J06cUO/evRUYGKigoCANGDBAp0+f9qrZunWr2rVrJ39/f4WHh+ull17K15f58+erQYMG8vf3V9OmTbVkyZLr3RwAAGCp6w45mZmZat68uaZNm3bZmk6dOunQoUPO65///KfX9N69e2vHjh1KTEzUokWLtHr1ag0aNMiZnpGRoY4dO6pmzZpKSUnRyy+/rHHjxmnmzJlOzdq1a9WrVy8NGDBAW7ZsUbdu3dStWzdt3779ejcJAABYyPd6Z+jcubM6d+58xRo/Pz+FhoYWOG3nzp1atmyZNm3apBYtWkiSXn/9dXXp0kWvvPKKwsLC9N577yk7O1tvv/223G63GjdurNTUVE2ePNkJQ1OnTlWnTp00cuRISdKf//xnJSYmKiEhQTNmzLjezQIAAJYpkmtyVq1apeDgYNWvX19DhgzR8ePHnWnr1q1TUFCQE3AkKSYmRj4+PtqwYYNTc8cdd8jtdjs1sbGx2r17t3744QenJiYmxmu9sbGxWrdu3WX7lZWVpYyMDK8XAACwU6GHnE6dOukf//iHkpKS9OKLLyo5OVmdO3dWTk6OJCktLU3BwcFe8/j6+qpy5cpKS0tzakJCQrxq8j5frSZvekEmTJggj8fjvMLDw3/exgIAgBLruk9XXU3Pnj2d902bNlWzZs108803a9WqVerQoUNhr+66jB49WiNGjHA+Z2RkEHQAALBUkd9CXrt2bVWpUkVff/21JCk0NFRHjhzxqjl//rxOnDjhXMcTGhqqw4cPe9Xkfb5azeWuBZIuXCsUGBjo9QIAAHYq8pDz3Xff6fjx46pWrZokKTo6WidPnlRKSopTs2LFCuXm5ioqKsqpWb16tc6dO+fUJCYmqn79+qpUqZJTk5SU5LWuxMRERUdHF/UmAQCAUuC6Q87p06eVmpqq1NRUSdLevXuVmpqq/fv36/Tp0xo5cqTWr1+vffv2KSkpSffdd5/q1Kmj2NhYSVLDhg3VqVMnDRw4UBs3btQXX3yhoUOHqmfPngoLC5MkPfTQQ3K73RowYIB27NihuXPnaurUqV6nmp544gktW7ZMkyZN0q5duzRu3Dht3rxZQ4cOLYRhAQAApd11h5zNmzfr1ltv1a233ipJGjFihG699VaNGTNGZcqU0datW3XvvfeqXr16GjBggCIjI/X555/Lz8/PWcZ7772nBg0aqEOHDurSpYvatm3r9Qwcj8ejTz/9VHv37lVkZKSefPJJjRkzxutZOq1bt9acOXM0c+ZMNW/eXB988IEWLFigJk2a/JzxAAAAlnAZY0xxd6K4ZGRkyOPxKD09vViuz4kYtfiqNfsmxt2AngCwRnam9MKFo+J65qDkLl+8/QGKwLV+f/PbVQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKvsXdAVxZxKjF+dr2TYwrhp4AAFC6cCQHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEq+xd2BX5KIUYuLuwsAAPxicCQHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFjpukPO6tWr1bVrV4WFhcnlcmnBggVe040xGjNmjKpVq6aAgADFxMRoz549XjUnTpxQ7969FRgYqKCgIA0YMECnT5/2qtm6davatWsnf39/hYeH66WXXsrXl/nz56tBgwby9/dX06ZNtWTJkuvdHAAAYKnrDjmZmZlq3ry5pk2bVuD0l156Sa+99ppmzJihDRs2qHz58oqNjdXZs2edmt69e2vHjh1KTEzUokWLtHr1ag0aNMiZnpGRoY4dO6pmzZpKSUnRyy+/rHHjxmnmzJlOzdq1a9WrVy8NGDBAW7ZsUbdu3dStWzdt3779ejcJAABYyGWMMT95ZpdLH330kbp16ybpwlGcsLAwPfnkk/q///s/SVJ6erpCQkI0e/Zs9ezZUzt37lSjRo20adMmtWjRQpK0bNkydenSRd99953CwsI0ffp0/fGPf1RaWprcbrckadSoUVqwYIF27dolSXrwwQeVmZmpRYsWOf1p1aqVbrnlFs2YMeOa+p+RkSGPx6P09HQFBgb+1GG4ZhGjFhfKcvZNjCuU5QCwUHam9ELYhffPHJTc5Yu3P0ARuNbv70K9Jmfv3r1KS0tTTEyM0+bxeBQVFaV169ZJktatW6egoCAn4EhSTEyMfHx8tGHDBqfmjjvucAKOJMXGxmr37t364YcfnJqL15NXk7eegmRlZSkjI8PrBQAA7FSoISctLU2SFBIS4tUeEhLiTEtLS1NwcLDXdF9fX1WuXNmrpqBlXLyOy9XkTS/IhAkT5PF4nFd4ePj1biIAACglflF3V40ePVrp6enO68CBA8XdJQAAUEQKNeSEhoZKkg4fPuzVfvjwYWdaaGiojhw54jX9/PnzOnHihFdNQcu4eB2Xq8mbXhA/Pz8FBgZ6vQAAgJ0KNeTUqlVLoaGhSkpKctoyMjK0YcMGRUdHS5Kio6N18uRJpaSkODUrVqxQbm6uoqKinJrVq1fr3LlzTk1iYqLq16+vSpUqOTUXryevJm89NosYtdjrBQAA8rvukHP69GmlpqYqNTVV0oWLjVNTU7V//365XC4NGzZMzz//vD7++GNt27ZNv/vd7xQWFubcgdWwYUN16tRJAwcO1MaNG/XFF19o6NCh6tmzp8LCLtwR8NBDD8ntdmvAgAHasWOH5s6dq6lTp2rEiBFOP5544gktW7ZMkyZN0q5duzRu3Dht3rxZQ4cO/fmjAgAASj3f651h8+bNuuuuu5zPecGjT58+mj17tp566illZmZq0KBBOnnypNq2batly5bJ39/fmee9997T0KFD1aFDB/n4+Kh79+567bXXnOkej0effvqp4uPjFRkZqSpVqmjMmDFez9Jp3bq15syZo2effVbPPPOM6tatqwULFqhJkyY/aSAAAIBdftZzckq70vqcnEvx3BwADp6Tg1+AYnlODgAAQElByAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwkm9xdwA/X8Soxfna9k2MK4aeAABQcnAkBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArFToIWfcuHFyuVxerwYNGjjTz549q/j4eN10002qUKGCunfvrsOHD3stY//+/YqLi1O5cuUUHByskSNH6vz58141q1at0m233SY/Pz/VqVNHs2fPLuxNAQAApViRHMlp3LixDh065LzWrFnjTBs+fLg++eQTzZ8/X8nJyTp48KB+85vfONNzcnIUFxen7OxsrV27Vn//+981e/ZsjRkzxqnZu3ev4uLidNdddyk1NVXDhg3To48+quXLlxfF5gAAgFLIt0gW6uur0NDQfO3p6en629/+pjlz5ujuu++WJM2aNUsNGzbU+vXr1apVK3366af66quv9NlnnykkJES33HKL/vznP+vpp5/WuHHj5Ha7NWPGDNWqVUuTJk2SJDVs2FBr1qzRlClTFBsbWxSbVOpEjFrs9XnfxLhi6gkAAMWjSI7k7NmzR2FhYapdu7Z69+6t/fv3S5JSUlJ07tw5xcTEOLUNGjRQjRo1tG7dOknSunXr1LRpU4WEhDg1sbGxysjI0I4dO5yai5eRV5O3jMvJyspSRkaG1wsAANip0ENOVFSUZs+erWXLlmn69Onau3ev2rVrp1OnTiktLU1ut1tBQUFe84SEhCgtLU2SlJaW5hVw8qbnTbtSTUZGhs6cOXPZvk2YMEEej8d5hYeH/9zNBQAAJVShn67q3Lmz875Zs2aKiopSzZo1NW/ePAUEBBT26q7L6NGjNWLECOdzRkYGQQcAAEsV+S3kQUFBqlevnr7++muFhoYqOztbJ0+e9Ko5fPiwcw1PaGhovrut8j5frSYwMPCKQcrPz0+BgYFeLwAAYKciDzmnT5/WN998o2rVqikyMlJly5ZVUlKSM3337t3av3+/oqOjJUnR0dHatm2bjhw54tQkJiYqMDBQjRo1cmouXkZeTd4yAAAACj3k/N///Z+Sk5O1b98+rV27Vvfff7/KlCmjXr16yePxaMCAARoxYoRWrlyplJQU9evXT9HR0WrVqpUkqWPHjmrUqJEeeeQR/fvf/9by5cv17LPPKj4+Xn5+fpKkwYMH69tvv9VTTz2lXbt26Y033tC8efM0fPjwwt4cAABQShX6NTnfffedevXqpePHj6tq1apq27at1q9fr6pVq0qSpkyZIh8fH3Xv3l1ZWVmKjY3VG2+84cxfpkwZLVq0SEOGDFF0dLTKly+vPn366LnnnnNqatWqpcWLF2v48OGaOnWqqlevrrfeeovbxwEAgMNljDHF3YnikpGRIY/Ho/T09Btyfc6lz665kXhODvALkZ0pvRB24f0zByV3+eLtD1AErvX7m9+uAgAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAl3+LuAG6Mgn4BnV8mBwDYjCM5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACv5FncHUHwiRi32+rxvYlwx9QQAgMLHkRwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWInfroLj0t+ykvg9KwBA6cWRHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlXhODq7o0mfn8NwcAEBpwZEcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABW4u4qXBd+qRwAUFpwJAcAAFiJkAMAAKxEyAEAAFbimhz8bDwVGQBQEnEkBwAAWIkjOUWkoLuQAADAjUPIQaHjNnMAQElAyMENwXU7AIAbjWtyAACAlUr9kZxp06bp5ZdfVlpampo3b67XX39dt99+e3F3C1fBKS0AQFEr1SFn7ty5GjFihGbMmKGoqCi9+uqrio2N1e7duxUcHFzc3cN14pQWAKAwuYwxprg78VNFRUWpZcuWSkhIkCTl5uYqPDxcjz/+uEaNGnXV+TMyMuTxeJSenq7AwMBC7Rt3V90YBCHgEtmZ0gthF94/c1Byly/e/gBF4Fq/v0vtkZzs7GylpKRo9OjRTpuPj49iYmK0bt26AufJyspSVlaW8zk9PV3ShcEqbLlZPxb6MpFfjeHzr1qzfXxsvrYmY5dftQYolbIzpaz///+uGRmSO6d4+wMUgbzv7asdpym1IefYsWPKyclRSEiIV3tISIh27dpV4DwTJkzQ+PHj87WHh4cXSR9RMnheLZwaoNSZGFbcPQCK1KlTp+TxeC47vdSGnJ9i9OjRGjFihPM5NzdXJ06c0E033SSXy1Vo68nIyFB4eLgOHDhQ6KfB8D+M843DWN8YjPONwTjfGEU5zsYYnTp1SmFhVw7ypTbkVKlSRWXKlNHhw4e92g8fPqzQ0NAC5/Hz85Ofn59XW1BQUFF1UYGBgfwDugEY5xuHsb4xGOcbg3G+MYpqnK90BCdPqX1OjtvtVmRkpJKSkpy23NxcJSUlKTo6uhh7BgAASoJSeyRHkkaMGKE+ffqoRYsWuv322/Xqq68qMzNT/fr1K+6uAQCAYlaqQ86DDz6oo0ePasyYMUpLS9Mtt9yiZcuW5bsY+Ubz8/PT2LFj850aQ+FinG8cxvrGYJxvDMb5xigJ41yqn5MDAABwOaX2mhwAAIArIeQAAAArEXIAAICVCDkAAMBKhBwAAGAlQk4RmDZtmiIiIuTv76+oqCht3LixuLtUqqxevVpdu3ZVWFiYXC6XFixY4DXdGKMxY8aoWrVqCggIUExMjPbs2eNVc+LECfXu3VuBgYEKCgrSgAEDdPr06Ru4FSXbhAkT1LJlS1WsWFHBwcHq1q2bdu/e7VVz9uxZxcfH66abblKFChXUvXv3fE8Y379/v+Li4lSuXDkFBwdr5MiROn/+/I3clBJv+vTpatasmfPU1+joaC1dutSZzjgXvokTJ8rlcmnYsGFOG+NcOMaNGyeXy+X1atCggTO9xI2zQaF6//33jdvtNm+//bbZsWOHGThwoAkKCjKHDx8u7q6VGkuWLDF//OMfzYcffmgkmY8++shr+sSJE43H4zELFiww//73v829995ratWqZc6cOePUdOrUyTRv3tysX7/efP7556ZOnTqmV69eN3hLSq7Y2Fgza9Yss337dpOammq6dOliatSoYU6fPu3UDB482ISHh5ukpCSzefNm06pVK9O6dWtn+vnz502TJk1MTEyM2bJli1myZImpUqWKGT16dHFsUon18ccfm8WLF5v//Oc/Zvfu3eaZZ54xZcuWNdu3bzfGMM6FbePGjSYiIsI0a9bMPPHEE04741w4xo4daxo3bmwOHTrkvI4ePepML2njTMgpZLfffruJj493Pufk5JiwsDAzYcKEYuxV6XVpyMnNzTWhoaHm5ZdfdtpOnjxp/Pz8zD//+U9jjDFfffWVkWQ2bdrk1CxdutS4XC7z/fff37C+lyZHjhwxkkxycrIx5sKYli1b1syfP9+p2blzp5Fk1q1bZ4y5EEZ9fHxMWlqaUzN9+nQTGBhosrKybuwGlDKVKlUyb731FuNcyE6dOmXq1q1rEhMTTfv27Z2QwzgXnrFjx5rmzZsXOK0kjjOnqwpRdna2UlJSFBMT47T5+PgoJiZG69atK8ae2WPv3r1KS0vzGmOPx6OoqChnjNetW6egoCC1aNHCqYmJiZGPj482bNhww/tcGqSnp0uSKleuLElKSUnRuXPnvMa5QYMGqlGjhtc4N23a1OsJ47GxscrIyNCOHTtuYO9Lj5ycHL3//vvKzMxUdHQ041zI4uPjFRcX5zWeEvtzYduzZ4/CwsJUu3Zt9e7dW/v375dUMse5VP+sQ0lz7Ngx5eTk5PtZiZCQEO3atauYemWXtLQ0SSpwjPOmpaWlKTg42Gu6r6+vKleu7NTgf3JzczVs2DC1adNGTZo0kXRhDN1ut4KCgrxqLx3ngv4OedPwP9u2bVN0dLTOnj2rChUq6KOPPlKjRo2UmprKOBeS999/X19++aU2bdqUbxr7c+GJiorS7NmzVb9+fR06dEjjx49Xu3bttH379hI5zoQc4BcuPj5e27dv15o1a4q7K9aqX7++UlNTlZ6erg8++EB9+vRRcnJycXfLGgcOHNATTzyhxMRE+fv7F3d3rNa5c2fnfbNmzRQVFaWaNWtq3rx5CggIKMaeFYzTVYWoSpUqKlOmTL4ryQ8fPqzQ0NBi6pVd8sbxSmMcGhqqI0eOeE0/f/68Tpw4wd/hEkOHDtWiRYu0cuVKVa9e3WkPDQ1Vdna2Tp486VV/6TgX9HfIm4b/cbvdqlOnjiIjIzVhwgQ1b95cU6dOZZwLSUpKio4cOaLbbrtNvr6+8vX1VXJysl577TX5+voqJCSEcS4iQUFBqlevnr7++usSuT8TcgqR2+1WZGSkkpKSnLbc3FwlJSUpOjq6GHtmj1q1aik0NNRrjDMyMrRhwwZnjKOjo3Xy5EmlpKQ4NStWrFBubq6ioqJueJ9LImOMhg4dqo8++kgrVqxQrVq1vKZHRkaqbNmyXuO8e/du7d+/32uct23b5hUoExMTFRgYqEaNGt2YDSmlcnNzlZWVxTgXkg4dOmjbtm1KTU11Xi1atFDv3r2d94xz0Th9+rS++eYbVatWrWTuz4V+KfMv3Pvvv2/8/PzM7NmzzVdffWUGDRpkgoKCvK4kx5WdOnXKbNmyxWzZssVIMpMnTzZbtmwx//3vf40xF24hDwoKMgsXLjRbt2419913X4G3kN96661mw4YNZs2aNaZu3brcQn6RIUOGGI/HY1atWuV1K+iPP/7o1AwePNjUqFHDrFixwmzevNlER0eb6OhoZ3reraAdO3Y0qampZtmyZaZq1arccnuJUaNGmeTkZLN3716zdetWM2rUKONyucynn35qjGGci8rFd1cZwzgXlieffNKsWrXK7N2713zxxRcmJibGVKlSxRw5csQYU/LGmZBTBF5//XVTo0YN43a7ze23327Wr19f3F0qVVauXGkk5Xv16dPHGHPhNvI//elPJiQkxPj5+ZkOHTqY3bt3ey3j+PHjplevXqZChQomMDDQ9OvXz5w6daoYtqZkKmh8JZlZs2Y5NWfOnDG///3vTaVKlUy5cuXM/fffbw4dOuS1nH379pnOnTubgIAAU6VKFfPkk0+ac+fO3eCtKdn69+9vatasadxut6latarp0KGDE3CMYZyLyqUhh3EuHA8++KCpVq2acbvd5le/+pV58MEHzddff+1ML2nj7DLGmMI/PgQAAFC8uCYHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFb6f8w+JS4bs3fgAAAAAElFTkSuQmCC\n"},"metadata":{}}]},{"cell_type":"markdown","source":["## 데이터셋 생성"],"metadata":{"id":"gIhm6s8lIGMu"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"BSswr5TKvoNM"},"outputs":[],"source":["train_batches = make_batches(train_examples, tokenizers=tokenizers,\n","                             MAX_TOKENS=MAX_TOKENS, BUFFER_SIZE=BUFFER_SIZE,\n","                             BATCH_SIZE=BATCH_SIZE)\n","val_batches = make_batches(val_examples, tokenizers=tokenizers,\n","                           MAX_TOKENS=MAX_TOKENS, BUFFER_SIZE=BUFFER_SIZE,\n","                           BATCH_SIZE=BATCH_SIZE)"]},{"cell_type":"markdown","source":["# 각 레이어 출력 크기"],"metadata":{"id":"Hqa59q2TLJb5"}},{"cell_type":"code","source":["for (pt, en), en_labels in train_batches.take(1):\n","    break\n","\n","print(pt.shape)\n","print(en.shape)\n","print(en_labels.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"grnv8XvHI-S-","executionInfo":{"status":"ok","timestamp":1721906697082,"user_tz":-540,"elapsed":1204,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"bd2f2b25-6b47-48e4-82e5-b50b25d049c5"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(64, 73)\n","(64, 69)\n","(64, 69)\n"]}]},{"cell_type":"code","source":["embed_pt = PositionalEmbedding(vocab_size=tokenizers.pt.get_vocab_size().numpy(), d_model=512)\n","embed_en = PositionalEmbedding(vocab_size=tokenizers.en.get_vocab_size().numpy(), d_model=512)\n","\n","pt_emb = embed_pt(pt)\n","en_emb = embed_en(en)"],"metadata":{"id":"Gq8umPn9LN4I"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["en_emb._keras_mask"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DxqE0XgkLRtX","executionInfo":{"status":"ok","timestamp":1721906697609,"user_tz":-540,"elapsed":6,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"b0ff67bc-3a6c-4eca-be23-3a7b229c197d"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tf.Tensor: shape=(64, 69), dtype=bool, numpy=\n","array([[ True,  True,  True, ..., False, False, False],\n","       [ True,  True,  True, ..., False, False, False],\n","       [ True,  True,  True, ..., False, False, False],\n","       ...,\n","       [ True,  True,  True, ..., False, False, False],\n","       [ True,  True,  True, ..., False, False, False],\n","       [ True,  True,  True, ..., False, False, False]])>"]},"metadata":{},"execution_count":33}]},{"cell_type":"code","source":["sample_ca = CrossAttention(num_heads=2, key_dim=512)\n","\n","print(pt_emb.shape)\n","print(en_emb.shape)\n","print(sample_ca(en_emb, pt_emb).shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YSIGkOqvLZ3E","executionInfo":{"status":"ok","timestamp":1721906698270,"user_tz":-540,"elapsed":665,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"e24aa14b-56f0-4357-ddd7-e94b3ef0368d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(64, 73, 512)\n","(64, 69, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'query' (of type EinsumDense) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'key' (of type EinsumDense) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'value' (of type EinsumDense) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 69, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_gsa = GlobalSelfAttention(num_heads=2, key_dim=512)\n","\n","print(pt_emb.shape)\n","print(sample_gsa(pt_emb).shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"p4suVxvJLcWj","executionInfo":{"status":"ok","timestamp":1721906699043,"user_tz":-540,"elapsed":788,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"6cb246e6-bb9e-456d-9bc3-3c41b882dbff"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(64, 73, 512)\n","(64, 73, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_csa = CausalSelfAttention(num_heads=2, key_dim=512)\n","\n","print(en_emb.shape)\n","print(sample_csa(en_emb).shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UlqTPp0PLfTO","executionInfo":{"status":"ok","timestamp":1721906699519,"user_tz":-540,"elapsed":486,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"cced3bf3-8ce7-4c8f-8c6e-fbd4bf52e8fd"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(64, 69, 512)\n","(64, 69, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_ffn = FeedForward(512, 2048)\n","\n","print(en_emb.shape)\n","print(sample_ffn(en_emb).shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZstqflFaLqKY","executionInfo":{"status":"ok","timestamp":1721906700299,"user_tz":-540,"elapsed":795,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"ee9e9197-4f1a-456d-ca57-87b437f36bfa"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 69, 512)\n","(64, 69, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_encoder_layer = EncoderLayer(d_model=512, num_heads=8, dff=2048)\n","\n","print(pt_emb.shape)\n","print(sample_encoder_layer(pt_emb).shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8RdApaSqLujL","executionInfo":{"status":"ok","timestamp":1721906703685,"user_tz":-540,"elapsed":3413,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"69963dc7-ff72-4405-e972-25b646cf6bc2"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_1' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 73, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_1' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_1' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 73, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_encoder = Encoder(num_layers=4,\n","                         d_model=512,\n","                         num_heads=8,\n","                         dff=2048,\n","                         vocab_size=8500)\n","\n","sample_encoder_output = sample_encoder(pt, training=False)\n","\n","print(pt.shape)\n","print(sample_encoder_output.shape)  # Shape `(batch_size, input_seq_len, d_model)`."],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BNfqcsGRL6Zv","executionInfo":{"status":"ok","timestamp":1721906723664,"user_tz":-540,"elapsed":19992,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"dd4692e2-e3a6-44d2-8086-85694164b087"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_2' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_2' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_2' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_1' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_3' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_3' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_3' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_2' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_4' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_4' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_4' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_3' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_5' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_5' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_5' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_4' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 73)\n","(64, 73, 512)\n"]}]},{"cell_type":"code","source":["sample_decoder_layer = DecoderLayer(d_model=512, num_heads=8, dff=2048)\n","sample_decoder_layer_output = sample_decoder_layer(x=en_emb, context=pt_emb)\n","\n","print(en_emb.shape)\n","print(pt_emb.shape)\n","print(sample_decoder_layer_output.shape)  # `(batch_size, seq_len, d_model)`"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KV-6GFoxL8aN","executionInfo":{"status":"ok","timestamp":1721906729362,"user_tz":-540,"elapsed":5702,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"933eddc7-6aca-4519-8805-32ed970cfe27"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_1' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_1' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_6' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_6' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 69, 512)\n","(64, 73, 512)\n","(64, 69, 512)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["sample_decoder = Decoder(num_layers=4,\n","                         d_model=512,\n","                         num_heads=8,\n","                         dff=2048,\n","                         vocab_size=8000)\n","\n","output = sample_decoder(x=en, context=pt_emb)\n","\n","print(en.shape)\n","print(pt_emb.shape)\n","print(output.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BipJcJYcMBWq","executionInfo":{"status":"ok","timestamp":1721906764776,"user_tz":-540,"elapsed":35419,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"0edb40e3-af5f-43ec-8f2e-127ab9dd5920"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_2' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_2' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_7' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_7' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_1' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_3' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_3' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_8' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_8' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_2' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_4' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_4' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_9' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_9' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_3' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_5' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_5' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_10' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_10' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_4' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 69)\n","(64, 73, 512)\n","(64, 69, 512)\n"]}]},{"cell_type":"code","source":["sample_decoder.last_attn_scores.shape  # (batch, heads, target_seq, input_seq)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cZruLCBuMSMP","executionInfo":{"status":"ok","timestamp":1721906764776,"user_tz":-540,"elapsed":23,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"d247e4fa-2fb4-40e2-b97b-b1bfa6c70ee5"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["TensorShape([64, 8, 69, 73])"]},"metadata":{},"execution_count":42}]},{"cell_type":"markdown","source":["# 모델"],"metadata":{"id":"PzD5nGofM4iQ"}},{"cell_type":"markdown","source":["## 트랜스포머"],"metadata":{"id":"rtBHDGA6MZ7w"}},{"cell_type":"markdown","source":["### 하이퍼 파라미터"],"metadata":{"id":"m6b766gTM7Nh"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"mzyo6KDfVyhl"},"outputs":[],"source":["num_layers = 4\n","d_model = 128\n","dff = 512\n","num_heads = 8\n","dropout_rate = 0.1"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"UiysUa--4tOU"},"outputs":[],"source":["transformer = Transformer(\n","                          num_layers=num_layers,\n","                          d_model=d_model,\n","                          num_heads=num_heads,\n","                          dff=dff,\n","                          input_vocab_size=tokenizers.pt.get_vocab_size().numpy(),\n","                          target_vocab_size=tokenizers.en.get_vocab_size().numpy(),\n","                          dropout_rate=dropout_rate)"]},{"cell_type":"code","source":["output = transformer((pt, en))\n","\n","print(en.shape)\n","print(pt.shape)\n","print(output.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kRygfIgCMdCl","executionInfo":{"status":"ok","timestamp":1721906776302,"user_tz":-540,"elapsed":11547,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"cc396fc9-3925-44c9-ecd0-7096da5e8ec9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_6' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_11' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_11' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_5' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_7' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_12' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_12' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_6' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_8' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_13' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_13' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_7' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'global_self_attention_9' (of type GlobalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_14' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_14' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'encoder_layer_8' (of type EncoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_6' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_6' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_15' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_15' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_5' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_7' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_7' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_16' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_16' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_6' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_8' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_8' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_17' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_17' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_7' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'causal_self_attention_9' (of type CausalSelfAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'cross_attention_9' (of type CrossAttention) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'sequential_18' (of type Sequential) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'feed_forward_18' (of type FeedForward) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/keras/src/layers/layer.py:915: UserWarning: Layer 'decoder_layer_8' (of type DecoderLayer) was passed an input with a mask attached to it. However, this layer does not support masking and will therefore destroy the mask information. Downstream layers will not see the mask.\n","  warnings.warn(\n"]},{"output_type":"stream","name":"stdout","text":["(64, 69)\n","(64, 73)\n","(64, 69, 7010)\n"]}]},{"cell_type":"code","source":["attn_scores = transformer.decoder.dec_layers[-1].last_attn_scores\n","print(attn_scores.shape)  # (batch, heads, target_seq, input_seq)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Y2y-xoP1Mg4O","executionInfo":{"status":"ok","timestamp":1721906776302,"user_tz":-540,"elapsed":21,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"5f7562d5-7373-48e8-a983-3b3160a7d929"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(64, 8, 69, 73)\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7r4scdulztRx"},"outputs":[],"source":["learning_rate = CustomSchedule(d_model=d_model)\n","optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,\n","                                     epsilon=1e-9)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Xij3MwYVRAAS","colab":{"base_uri":"https://localhost:8080/","height":449},"executionInfo":{"status":"ok","timestamp":1721906776989,"user_tz":-540,"elapsed":706,"user":{"displayName":"Seung Hoon Lee","userId":"06197787624635186839"}},"outputId":"976e095d-41ba-4297-e877-9b9df2eb1623"},"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAlEAAAGwCAYAAACJjDBkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABq9ElEQVR4nO3deVxU5f4H8M8MMDOsMyDCgCKg4o5LLohpVlJYZtJtUa83zetPu/20Mq1Mr2J17Vpm95ZlWbfF+rW4dMvK1CLcUhEVwRVxQ8Bl2GHYl5nn9wdydBIVcIbDDJ/36zUv5JzvOef7MOh8fZ7nPEchhBAgIiIioiZRyp0AERERkT1iEUVERETUDCyiiIiIiJqBRRQRERFRM7CIIiIiImoGFlFEREREzcAiioiIiKgZnOVOwJGZzWZcvHgRnp6eUCgUcqdDREREjSCEQElJCQIDA6FUXr+/iUWUDV28eBFBQUFyp0FERETNkJWVhY4dO153P4soG/L09ARQ9yZ4eXnJnA0RERE1htFoRFBQkPQ5fj0somyofgjPy8uLRRQREZGdudlUHE4sJyIiImoGFlFEREREzcAiioiIiKgZWEQRERERNQOLKCIiIqJmYBFFRERE1AwsooiIiIiagUUUERERUTOwiCIiIiJqBhZRRERERM0gexG1cuVKhISEQKPRICIiAvv27bth/Pr169GjRw9oNBqEh4dj06ZNFvuFEIiNjUVAQABcXV0RFRWFU6dOWcS89tprGDZsGNzc3KDT6W54vfz8fHTs2BEKhQJFRUXNaSIRERE5IFmLqLVr12LOnDlYvHgxDh48iH79+iE6Oho5OTkNxu/ZswcTJ07EtGnTkJycjJiYGMTExODo0aNSzLJly7BixQqsWrUKiYmJcHd3R3R0NCorK6WY6upqPProo3jqqadumuO0adPQt2/fW28sERERORSFEELIdfGIiAgMHjwY7733HgDAbDYjKCgITz/9NF566aVr4sePH4+ysjJs3LhR2jZ06FD0798fq1atghACgYGBmDt3Lp5//nkAQHFxMfz9/bF69WpMmDDB4nyrV6/G7Nmzr9vD9MEHH2Dt2rWIjY3FqFGjUFhYeMOeq6qqKlRVVUnf1z8Furi4uM0/gFgIAZNZwNlJ9s5PIiKiGzIajdBqtTf9/JbtE626uhpJSUmIioq6koxSiaioKCQkJDR4TEJCgkU8AERHR0vx6enpMBgMFjFarRYRERHXPef1HD9+HK+++iq++OILKJWN+zEtXboUWq1WegUFBTXpmo5s1tfJGLo0HjkllTcPJiIisgOyFVF5eXkwmUzw9/e32O7v7w+DwdDgMQaD4Ybx9V+bcs6GVFVVYeLEiXjzzTfRqVOnRh83f/58FBcXS6+srKxGH+vIhBD4+cgl5JVW45Nd6XKnQ0REZBXOcifQGs2fPx89e/bEX/7ylyYdp1aroVarbZSV/copuTLEedJQImMmRERE1iNbT5Svry+cnJyQnZ1tsT07Oxt6vb7BY/R6/Q3j67825ZwN2bp1K9avXw9nZ2c4Oztj1KhRUs6LFy9u9HmoTmZBufTnfekFqK41y5gNERGRdchWRKlUKgwcOBDx8fHSNrPZjPj4eERGRjZ4TGRkpEU8AMTFxUnxoaGh0Ov1FjFGoxGJiYnXPWdD/vvf/+LQoUNISUlBSkoKPv74YwDA77//jpkzZzb6PFQnM/9KEVVWbcLBzEIZsyEiIrIOWYfz5syZgylTpmDQoEEYMmQI3n77bZSVlWHq1KkAgMmTJ6NDhw5YunQpAODZZ5/FyJEj8dZbb2HMmDFYs2YNDhw4gI8++ggAoFAoMHv2bCxZsgRhYWEIDQ3FokWLEBgYiJiYGOm6mZmZKCgoQGZmJkwmE1JSUgAAXbt2hYeHB7p06WKRZ15eHgCgZ8+eN11Xiq6VcVVPFADsOJmLoZ3byZQNERGRdchaRI0fPx65ubmIjY2FwWBA//79sWXLFmlieGZmpsWdccOGDcPXX3+NhQsXYsGCBQgLC8OGDRvQp08fKebFF19EWVkZZsyYgaKiIgwfPhxbtmyBRqORYmJjY/H5559L3w8YMAAAsG3bNtx55502bnXbk3W5iOru74m07BLsSMvFvNE9ZM6KiIjo1si6TpSja+w6E47u4Q/2ICmjEK+O643FPx6DEMC+BaPg56W5+cFEREQtrNWvE0VtR8blOVEDgrwR3kELANh5Kk/OlIiIiG4ZiyiyqfLqWuSV1i1x0MnHDSO7tQdQNy+KiIjInrGIIpvKKqgAAGhdXaB1c5GKqN9P5aLWxKUOiIjIfrGIIpvKyC8DUNcLBQD9g3TQubmgqLwGSRlc6oCIiOwXiyiyqfqFNuuLKGcnJe7u7gcA+C01+7rHERERtXYsosimpCKqnZu0LapX3RIWccezwZtDiYjIXrGIIpv6Y08UANzRrT1UTkqcyy/HmdxSuVIjIiK6JSyiyKYaKqI81M6I7FK3YvmvxzmkR0RE9olFFNmMySxw/vLdeVcXUQBwz+Uhvd9YRBERkZ1iEUU2k22sRLXJDGelAgFay9XJR/Wsm1yenFWE3JIqOdIjIiK6JSyiyGbqh/I6eLvC2cnyVy1A64rwDloIAWw9wd4oIiKyPyyiyGYy86+dD3W1e666S4+IiMjesIgim2loUvnV7u1dV0TtPJWHksqaFsuLiIjIGlhEkc3crIjq7u+Jzu3dUV1rRnxqTkumRkREdMtYRJHNZFwuooLbNVxEKRQKjAkPAAD8fORSi+VFRERkDSyiyGayLhdRQdfpiQKAMX3riqgdJ3M5pEdERHaFRRTZREllDQrKqgFcfzgP4JAeERHZLxZRZBP186F83FXw1LhcN45DekREZK9YRJFNNGYorx6H9IiIyB6xiCKbqO+JCm5EEcUhPSIiskcsosgmMm6y0ObVrh7S++nQRZvmRUREZC0sosgmbrZG1B892C8QQN2QXn4pn6VHREStH4sosgmpiLrOGlF/FObviT4dvFBrFpxgTkREdoFFFFldrcmMC4UVABrfEwUADw3oCAD47uAFm+RFRERkTSyiyOouFVei1iygclLC30vT6OMe7BcIJ6UCKVlFSM8rs2GGREREt45FFFld/VBeRx9XOCkVjT6uvacaw7v6AgC+T2ZvFBERtW4sosjqmjqp/Gp/uq0DAGBD8gUIIayaFxERkTWxiCKru5Ui6p5e/nBTOSGzoBwHMwutnRoREZHVsIgiq8tswhpRf+SmcsboPnoAwH85wZyIiFoxFlFkdbfSEwUAj9xWd5feTykXUV5da7W8iIiIrIlFFFmd9MiXdu7NOn5o53bo5OOGkqpa/HyYa0YREVHrxCKKrKq4vAbFFXUPEQ7ycW3WOZRKBcYPDgIArN2fZbXciIiIrIlFFFlVfS+Ur4cabirnZp/n0YEd4aRU4EBGIU5ll1grPSIiIqthEUVWdWUor3nzoer5eWlwdw8/AOyNIiKi1olFFFlVRkHdSuPNnVR+tQmXh/S+S76AqlrTLZ+PiIjImlhEkVVlXe6JCrJCETWyW3vovTQoKKtG3PHsWz4fERGRNbGIIqvKuLxGVLAViihnJyUeHVS33MHXiZm3fD4iIiJrkr2IWrlyJUJCQqDRaBAREYF9+/bdMH79+vXo0aMHNBoNwsPDsWnTJov9QgjExsYiICAArq6uiIqKwqlTpyxiXnvtNQwbNgxubm7Q6XTXXOPQoUOYOHEigoKC4Orqip49e+Kdd9655ba2BdIaUbc4J6rehCGdoFQAe87kc4I5ERG1KrIWUWvXrsWcOXOwePFiHDx4EP369UN0dDRycnIajN+zZw8mTpyIadOmITk5GTExMYiJicHRo0elmGXLlmHFihVYtWoVEhMT4e7ujujoaFRWVkox1dXVePTRR/HUU081eJ2kpCT4+fnhyy+/xLFjx/D3v/8d8+fPx3vvvWfdH4CDqTGZcbGoAoB15kQBQAedK+7p5Q8A+CIhwyrnJCIisgaFkPEprxERERg8eLBUnJjNZgQFBeHpp5/GSy+9dE38+PHjUVZWho0bN0rbhg4div79+2PVqlUQQiAwMBBz587F888/DwAoLi6Gv78/Vq9ejQkTJlicb/Xq1Zg9ezaKiopumuvMmTORmpqKrVu3XjemqqoKVVVV0vdGoxFBQUEoLi6Gl5fXTa9h787lleHO5duhdlbixD9GQ6FQWOW8e07n4c8fJ8JN5YS9C0bBS+NilfMSERE1xGg0QqvV3vTzW7aeqOrqaiQlJSEqKupKMkoloqKikJCQ0OAxCQkJFvEAEB0dLcWnp6fDYDBYxGi1WkRERFz3nI1VXFwMHx+fG8YsXboUWq1WegUFBd3SNe3N1Y97sVYBBQCRXdohzM8D5dUm/DfpvNXOS0REdCtkK6Ly8vJgMpng7+9vsd3f3x8Gg6HBYwwGww3j67825ZyNsWfPHqxduxYzZsy4Ydz8+fNRXFwsvbKy2tb6Rrf6zLzrUSgUmDwsBEDdkJ7ZLFvnKRERkUT2ieWt3dGjRzFu3DgsXrwY99577w1j1Wo1vLy8LF5tibUnlV/tTwM6wFPtjPS8Mvx+Os/q5yciImoq2YooX19fODk5ITvbcv2f7Oxs6PX6Bo/R6/U3jK//2pRz3sjx48cxatQozJgxAwsXLmzy8W1NZr5teqIAwF3tjEcuL3ewene61c9PRETUVLIVUSqVCgMHDkR8fLy0zWw2Iz4+HpGRkQ0eExkZaREPAHFxcVJ8aGgo9Hq9RYzRaERiYuJ1z3k9x44dw1133YUpU6bgtddea9KxbZW1HvlyPZMjQ6BQANvScrncARERyU7W4bw5c+bgP//5Dz7//HOkpqbiqaeeQllZGaZOnQoAmDx5MubPny/FP/vss9iyZQveeustnDhxAi+//DIOHDiAWbNmAaibOzN79mwsWbIEP/74I44cOYLJkycjMDAQMTEx0nkyMzORkpKCzMxMmEwmpKSkICUlBaWlpQDqhvDuuusu3HvvvZgzZw4MBgMMBgNyc3Nb7odjZ4QQNpsTVS/U1x33Xl7u4D+/n7XJNYiIiBrLWc6Ljx8/Hrm5uYiNjYXBYED//v2xZcsWaWJ4ZmYmlMordd6wYcPw9ddfY+HChViwYAHCwsKwYcMG9OnTR4p58cUXUVZWhhkzZqCoqAjDhw/Hli1boNFopJjY2Fh8/vnn0vcDBgwAAGzbtg133nknvv32W+Tm5uLLL7/El19+KcUFBwfj3Llztvpx2LXC8hqUVtUCADp626aIAoAZd3TBL8eysSH5Ip6/tzv8vDQ3P4iIiMgGZF0nytE1dp0JR5CSVYSYlbuh99Jg74JRNr3WIx/swYGMQjx1ZxfMG93DptciIqK2p9WvE0WOJSO/DIDthvKuNuOOzgCAL/dmSL1fRERELY1FFFlF1uX5UEEtUERF9fRH5/buKKmsxZp9fDAxERHJg0UUWUVGvm3vzLuaUqnAjBF1vVGf7kpHjcls82sSERH9EYsosgpb35n3RzEDOqC9pxoXiyvx/cELLXJNIiKiq7GIIqtoyeE8ANC4OOHJy3Oj3tt2mr1RRETU4lhE0S2rqjXhkrESQMsM59X7c0QntHNXIbOgHD+kXGyx6xIREQEsosgKzhdWQAjATeWEdu6qFruum8oZ0y/3Rq3cdhq17I0iIqIWxCKKbtnV86EUCkWLXvvxocHwdnNBel4ZNh6+1KLXJiKito1FFN0yWz54+Gbc1c74n8t36q3YegomM9eOJSKilsEiim5ZS9+Z90eTI4OhdXXB2dwybDzMuVFERNQyWETRLZOKqBacVH41T40Lpo8IBQD8K+4k79QjIqIWwSKKbpmcw3n1pt4eCl8PFTLyy7F2f5ZseRARUdvBIopuiRBC9uE8oG5u1NN3hwEAVsSfQkW1SbZciIiobWARRbckr7QaFTUmKBRAR2/5iigAmDikEzp6uyKnpAqr95yTNRciInJ8LKLolmQWlAEAArWuUDnL++ukclZizj3dAAAfbD+N4vIaWfMhIiLHxiKKbkmm9LgXV5kzqTOufwd09/eEsbIWH+w4I3c6RETkwFhE0S3JuDypPNjHXeZM6jgpFXghujsA4LPd6ThfWC5zRkRE5KhYRNEtkXt5g4aM6umHoZ19UFVrxhtb0uROh4iIHBSLKLolWdJwXuspohQKBRY90AsKBfDToYtIyiiQOyUiInJALKLollwZzms9RRQA9A7UYvygIADAqz8dh5mPgyEiIitjEUXNVlFtQk5JFQB514i6nrn3doeH2hmHzhdjQ8oFudMhIiIHwyKKmq1+0ran2hk6NxeZs7lWe081Zt7VFQDwxpYTKKuqlTkjIiJyJCyiqNnqh/I6tXODQqGQOZuGTb09BEE+rsg2VuH97aflToeIiBwIiyhqttbwuJeb0bg4YeGYXgCAj3aexemcEpkzIiIiR8EiiprNHoooALi3lz/u7uGHGpPAwg1HIQQnmRMR0a1jEUXN1hrXiGqIQqHAKw/2hsZFib1nCzjJnIiIrIJFFDWbvfREAXXrWD19dxgA4LWfU/lcPSIiumUsoqhZzGYhLbTZWh75cjPTR3RGl/buyCutxpu/npA7HSIisnMsoqhZckqqUFVrhpNSgQCdRu50GkXlrMSSmHAAwFeJmUjKKJQ5IyIismcsoqhZ6ofyAnUauDjZz69RZJd2ePi2jhACmPffw6isMcmdEhER2Sn7+fSjViXTzobyrrbogZ5o76nG6ZxSrIg/JXc6RERkp1hEUbNk5pcBaF0PHm4snZsKS2L6AAA+3HkWR84Xy5wRERHZIxZR1Cz2dGdeQ6J76/FA3wCYzAIvfHsI1bVmuVMiIiI7wyKKmiWjfjivla8RdSOvPNgbPu4qnDCU8JEwRETUZCyiqFmy7LwnCgDaeajxyoO9AQDvbT2Nw+eL5E2IiIjsCosoarKyqlrklVYDsM85UVd7oG8A7g/Xo9YsMHtNCsqra+VOiYiI7ASLKGqy+vlQOjcXaF1dZM7m1igUCvzzoXD4e6lxNq8Mr/2cKndKRERkJ2QvolauXImQkBBoNBpERERg3759N4xfv349evToAY1Gg/DwcGzatMlivxACsbGxCAgIgKurK6KionDqlOVt7K+99hqGDRsGNzc36HS6Bq+TmZmJMWPGwM3NDX5+fnjhhRdQW8teCsD+J5X/kc5Nhbce7Q+gbhHO+NRseRMiIiK7IGsRtXbtWsyZMweLFy/GwYMH0a9fP0RHRyMnJ6fB+D179mDixImYNm0akpOTERMTg5iYGBw9elSKWbZsGVasWIFVq1YhMTER7u7uiI6ORmVlpRRTXV2NRx99FE899VSD1zGZTBgzZgyqq6uxZ88efP7551i9ejViY2Ot+wOwU/Xzoex9KO9qw8N88T/DQwEAL357GLklVTJnRERErZ6Q0ZAhQ8TMmTOl700mkwgMDBRLly5tMP6xxx4TY8aMsdgWEREhnnzySSGEEGazWej1evHmm29K+4uKioRarRbffPPNNef77LPPhFarvWb7pk2bhFKpFAaDQdr2wQcfCC8vL1FVVdXo9hUXFwsAori4uNHH2IOF3x8RwfM2ijc2p8qdilVVVNeK6H/vEMHzNoonPk0UZrNZ7pSIiEgGjf38lq0nqrq6GklJSYiKipK2KZVKREVFISEhocFjEhISLOIBIDo6WopPT0+HwWCwiNFqtYiIiLjuOa93nfDwcPj7+1tcx2g04tixY9c9rqqqCkaj0eLliBxtOK+exsUJ70wYAJWzEtvScvGf38/KnRIREbVishVReXl5MJlMFoUKAPj7+8NgMDR4jMFguGF8/demnLMp17n6Gg1ZunQptFqt9AoKCmr0Ne2JtLyBHa8RdT3d9Z5YPLYXAOCNLWk4cK5A5oyIiKi1kn1iuSOZP38+iouLpVdWVpbcKVmdySyQVeiYPVH1/jykEx7sFwiTWWDW18koKKuWOyUiImqFZCuifH194eTkhOxsyzuhsrOzodfrGzxGr9ffML7+a1PO2ZTrXH2NhqjVanh5eVm8HI3BWIkak4CLkwIBWle507EJhUKBf/4pHJ193WEwVuK5tSkwm4XcaRERUSsjWxGlUqkwcOBAxMfHS9vMZjPi4+MRGRnZ4DGRkZEW8QAQFxcnxYeGhkKv11vEGI1GJCYmXvec17vOkSNHLO4SjIuLg5eXF3r16tXo8ziizPy6XqiO3m5wUipkzsZ2PNTOWDnpNqidldhxMhcf7Dgjd0pERNTKyDqcN2fOHPznP//B559/jtTUVDz11FMoKyvD1KlTAQCTJ0/G/Pnzpfhnn30WW7ZswVtvvYUTJ07g5ZdfxoEDBzBr1iwAdT0Is2fPxpIlS/Djjz/iyJEjmDx5MgIDAxETEyOdJzMzEykpKcjMzITJZEJKSgpSUlJQWloKALj33nvRq1cvPP744zh06BB++eUXLFy4EDNnzoRarW65H1ArlFlQBsCxlje4np4BXnh1XN1jYd76NQ07TubKnBEREbUmznJefPz48cjNzUVsbCwMBgP69++PLVu2SJO4MzMzoVReqfOGDRuGr7/+GgsXLsSCBQsQFhaGDRs2oE+fPlLMiy++iLKyMsyYMQNFRUUYPnw4tmzZAo1GI8XExsbi888/l74fMGAAAGDbtm2488474eTkhI0bN+Kpp55CZGQk3N3dMWXKFLz66qu2/pG0elfuzHPMobw/emxQEA5mFGHtgSw8/fVB/DhrOEJ83eVOi4iIWgGFEIKTPWzEaDRCq9WiuLjYYeZHzfr6IDYevoS/398T0+/oLHc6LaKq1oQJH+1FcmYRwvw88P3M2+GhlvX/H0REZEON/fzm3XnUJI64WvnNqJ2dsOovA+HnqcapnFLM4URzIiICiyhqIkddaPNm/L00WPX4QKiclPj1eDbe3Xpa7pSIiEhmLKKo0YyVNSgsrwHgmAtt3sxtnbyxJKZu/t2/fzuJnw9fkjkjIiKSE4soarT65Q3auava7JygxwYH4YlhIQCA59alICmjUN6EiIhINiyiqNHa4nyohix6oBeievqhutaM6V8cQEZ+mdwpERGRDFhEUaNlXC6igtvgUN7VnJQKvDNhAPp08EJBWTWmrt6PonI+GoaIqK1hEUWN1lYnlTfEXe2MT6YMRqBWg7O5ZZjxf0moqjXJnRYREbUgFlHUaBzOs+TvpcGnUwfDQ+2MfekFmLPuEExc+oCIqM1gEUWNlnF5YnkwiyhJD70XPvjLbXBxUuDnw5cQ+8NRcP1aIqK2gUUUNUqtyYwLRRUA2ubyBjcyIqw9/vVYfygUwFeJmfh33Em5UyIiohbAIooa5VJxJUxmAZWzEv6empsf0MaM7ReIV8fVrSG1YutpfLY7XeaMiIjI1lhEUaPUD+UFebtCqVTInE3r9PjQYMy9pxsA4JWfjuP75PMyZ0RERLbEIooahXfmNc6su7ti6u0hAIDn1x/G5iNc1ZyIyFGxiKJGySioW1AyuJ27zJm0bgqFAovG9MLDt3WEySzw9DfJ+OWYQe60iIjIBlhEUaNweYPGUyoVWPZIX4zrH4has8Csrw/it+PZcqdFRERWxiKKGoXDeU3jpFTgrUf74YG+AagxCfzvVwex7USO3GkREZEVsYiimxJCXFkjissbNJqzkxJvj++P+8P1qDaZ8eSXSdiexkKKiMhRsIiimyquqEFJZS0AIMibRVRTODsp8c6EAYju7Y/qWjNmfJHEOVJERA7iloqoyspKa+VBrVj9UF57TzVcVU4yZ2N/XJyUeHfibVKP1P9+dRAbki/InRYREd2iJhdRZrMZ//jHP9ChQwd4eHjg7NmzAIBFixbhk08+sXqCJD8+7uXWqZyVWDFhgHTX3nPrUvB1YqbcaRER0S1ochG1ZMkSrF69GsuWLYNKpZK29+nTBx9//LFVk6PWgZPKrcPZSYk3H+mLx4cGQwhgwfdH8PHvZ+VOi4iImqnJRdQXX3yBjz76CJMmTYKT05WhnX79+uHEiRNWTY5aBy5vYD1KpQKvjuuNJ0d2BgAs+TkV//o1jQ8tJiKyQ00uoi5cuICuXbtes91sNqOmpsYqSVHrwjvzrEuhUOCl0T2kR8Ss2HoaL/33CGpNZpkzIyKipmhyEdWrVy/8/vvv12z/9ttvMWDAAKskRa0Lh/OsT6FQ4OlRYfjnQ+FQKoC1B7Iw/YsDKK+ulTs1IiJqJOemHhAbG4spU6bgwoULMJvN+O6775CWloYvvvgCGzdutEWOJKPqWjMuFVcAADqxJ8rq/hzRCe091Xj6m4PYlpaLiR/txSdPDIavh1ru1IiI6Caa3BM1btw4/PTTT/jtt9/g7u6O2NhYpKam4qeffsI999xjixxJRheKKmAWgMZFifb8YLeJe3r54+vpQ+Ht5oJD54vx8Ad7cC6vTO60iIjoJprcEwUAI0aMQFxcnLVzoVbo6qE8hUIhczaO67ZO3vj2qWGY8uk+ZOSXI+b93Vj1l4EY2rmd3KkREdF1NLknqnPnzsjPz79me1FRETp37myVpKj1uFJEucuciePr0t4D3/3vMPTtqEVReQ3+8nEi1uzjWlJERK1Vk4uoc+fOwWQyXbO9qqoKFy5wFWZHk5lfN6zESeUtw89Tg7UzIjGmbwBqzQIvfXcE/9h4HCYzl0AgImptGj2c9+OPP0p//uWXX6DVaqXvTSYT4uPjERISYtXkSH5XeqJcZc6k7XBVOeG9iQMQ5ueBt387hU92peNsbilWTBwAT42L3OkREdFljS6iYmJiANTdmj1lyhSLfS4uLggJCcFbb71l1eRIflfWiOJwXktSKBSYHdUNYX6emLs+BdvScvHQ+3vw4eMD0aW9h9zpERERmjCcZzabYTab0alTJ+Tk5Ejfm81mVFVVIS0tDQ888IAtc6UWJoTgauUyG9M3AOuejIS/lxqnc0ox7r3d2HLUIHdaRESEZsyJSk9Ph6+vry1yoVamoKwaZdUmKBRAR28O58mlb0cdfnp6OIaE+qC0qhZ/+zIJr28+wRXOiYhk1qwlDsrKyrBjxw5kZmaiurraYt8zzzxjlcRIfhmXe6H0XhpoXJxuEk225OepwVf/E4E3Np/Ax7vSsWrHGRw+X4QVEwdwYU4iIpk0uYhKTk7G/fffj/LycpSVlcHHxwd5eXlwc3ODn58fiygHwqG81sXFSYmFD/RC/046vPjtYew5k4+x7+7Cykm34bZO3nKnR0TU5jR5OO+5557D2LFjUVhYCFdXV+zduxcZGRkYOHAgli9fboscSSaZ+XxmXmv0QN9A/DDzdnRu745LxZV4bFUCPth+BmYug0BE1KKaXESlpKRg7ty5UCqVcHJyQlVVFYKCgrBs2TIsWLDAFjmSTOqH84JZRLU6Yf6e+GHm7Xjg8npSb2w5gSmf7UNOSaXcqRERtRlNLqJcXFygVNYd5ufnh8zMuhWVtVotsrKympzAypUrERISAo1Gg4iICOzbt++G8evXr0ePHj2g0WgQHh6OTZs2WewXQiA2NhYBAQFwdXVFVFQUTp06ZRFTUFCASZMmwcvLCzqdDtOmTUNpaalFzC+//IKhQ4fC09MT7du3x8MPP4xz5841uX32TFojig8ebpU8NS54d+IAvPFwODQuSvx+Kg/3v/M7dpzMlTs1IqI2oclF1IABA7B//34AwMiRIxEbG4uvvvoKs2fPRp8+fZp0rrVr12LOnDlYvHgxDh48iH79+iE6Oho5OTkNxu/ZswcTJ07EtGnTkJycjJiYGMTExODo0aNSzLJly7BixQqsWrUKiYmJcHd3R3R0NCorr/wPfdKkSTh27Bji4uKwceNG7Ny5EzNmzJD2p6enY9y4cbj77ruRkpKCX375BXl5efjTn/7UpPbZu6wCDue1dgqFAuMHd8LGp4ejh94TeaXVmPLpPvxzUyqqa3n3HhGRTYkm2r9/v9i6dasQQojs7GwRHR0tPD09xW233SaSk5ObdK4hQ4aImTNnSt+bTCYRGBgoli5d2mD8Y489JsaMGWOxLSIiQjz55JNCCCHMZrPQ6/XizTfflPYXFRUJtVotvvnmGyGEEMePHxcAxP79+6WYzZs3C4VCIS5cuCCEEGL9+vXC2dlZmEwmKebHH38UCoVCVFdXN7p9xcXFAoAoLi5u9DGtRUV1rQh5aaMInrdR5JVUyp0ONUJFda1YtOGICJ5X977d/85OceKSUe60iIjsTmM/v5vcEzVo0CDcddddAOqG87Zs2QKj0YikpCT079+/0eeprq5GUlISoqKipG1KpRJRUVFISEho8JiEhASLeACIjo6W4tPT02EwGCxitFotIiIipJiEhATodDoMGjRIiomKioJSqURiYiIAYODAgVAqlfjss89gMplQXFyM//u//0NUVBRcXK7/2I2qqioYjUaLl706X1gBIQB3lRN83FVyp0ONoHFxwqvj+uDDxwfC280Fxy4aMfbdXfhwxxk+e4+IyAaaXERdz8GDB5u0YnleXh5MJhP8/f0ttvv7+8NgaHhFZoPBcMP4+q83i/Hz87PY7+zsDB8fHykmNDQUv/76KxYsWAC1Wg2dTofz589j3bp1N2zT0qVLodVqpVdQUNAN41szaSivnTsUCoXM2VBTRPfW45fn7sCoHn6oNpmxdPMJTPgoQbrbkoiIrKNJRdQvv/yC559/HgsWLMDZs2cBACdOnEBMTAwGDx4Ms9kx5mAYDAZMnz4dU6ZMwf79+7Fjxw6oVCo88sgjEOL6/6OfP38+iouLpVdzJtq3Fhn5ZQD44GF75eepwcdTBuGNh8PhrnLC/nOFGP3OTnyVmHHD32EiImq8Ri+2+cknn2D69Onw8fFBYWEhPv74Y/zrX//C008/jfHjx+Po0aPo2bNnoy/s6+sLJycnZGdnW2zPzs6GXq9v8Bi9Xn/D+Pqv2dnZCAgIsIipH2rU6/XXTFyvra1FQUGBdPzKlSuh1WqxbNkyKebLL79EUFAQEhMTMXTo0AbzU6vVUKsdY/XozIIKAJxUbs/qJ50P6+KLuesPYV96Af7+/VFsPmLAPx8K512XRES3qNE9Ue+88w7eeOMN5OXlYd26dcjLy8P777+PI0eOYNWqVU0qoABApVJh4MCBiI+Pl7aZzWbEx8cjMjKywWMiIyMt4gEgLi5Oig8NDYVer7eIMRqNSExMlGIiIyNRVFSEpKQkKWbr1q0wm82IiIgAAJSXl0vLONRzcnKScmwLMgsu90S1c5c5E7pVQT5uWDN9KBaO6Qm1sxK7Tufh3rd34D87z/L5e0REt6KxM9Xd3NxEenq6EKLuLjgXFxexa9euW5j7LsSaNWuEWq0Wq1evFsePHxczZswQOp1OGAwGIYQQjz/+uHjppZek+N27dwtnZ2exfPlykZqaKhYvXixcXFzEkSNHpJjXX39d6HQ68cMPP4jDhw+LcePGidDQUFFRUSHFjB49WgwYMEAkJiaKXbt2ibCwMDFx4kRpf3x8vFAoFOKVV14RJ0+eFElJSSI6OloEBweL8vLyRrfPnu/Ou+df20XwvI1ie1qO3KmQFZ3NLRXjP9wj3cE39t3fxbEL9vf7SURkS439/G50EaVQKER2drb0vYeHhzhz5kzzM7zs3XffFZ06dRIqlUoMGTJE7N27V9o3cuRIMWXKFIv4devWiW7dugmVSiV69+4tfv75Z4v9ZrNZLFq0SPj7+wu1Wi1GjRol0tLSLGLy8/PFxIkThYeHh/Dy8hJTp04VJSUlFjHffPONGDBggHB3dxft27cXDz74oEhNTW1S2+y1iDKbzaL7wk0ieN5GcTa3VO50yMrMZrP4JjFD9Fm8RQTP2yi6zP9ZLNuSKiqqa+VOjYioVWjs57dCiMbNMlUqlViyZAk8PDwAAPPmzcMLL7wAX19fizg+gPgKo9EIrVaL4uJieHl5yZ1Oo+UYKzHkn/FQKoAT/7gPKmer3cRJrUiOsRKLfzyGzUfr7koNaeeGlx/sjTu7+93kSCIix9bYz+9GF1EhISE3vdVdoVBId+2R/RZRB84V4JFVCeigc8Xul+6WOx2ysS1HDYj94ShySqoAAKN767FobC900PHOTCJqmxr7+d3ou/Pa2nPj2rJMPu6lTRndR4/bu7bDO7+dwmd7zmHLMQO2n8zB03eH4X9GhELt7CR3ikRErRLHaegaGZcXZQzmLfBthqfGBQsf6IVNz4zAkFAfVNaY8eYvabjv7d/x+yk+0JiIqCEsouga9auVB7Enqs3prvfE2hlD8fb4/vD1UONsXhke/2Qf/ufzAzibWyp3ekRErQqLKLpG/XAee6LaJoVCgZgBHbD1+ZGYensInJQK/JaajXv/vROv/HQMReXVcqdIRNQqsIiia2RwThQB8NK4YPHY3vhl9h24u4cfas0Cn+0+h5Fvbsenu9JRXcuFOomobWMRRRYqqk3IvXyXFosoAoCufh749InB+L9pQ9BD74niihq8uvE4ot/eiV+PGfgsPiJqsxp9d149o9HY4HaFQgG1Wg2VSnXLSZF8sgrreqG8NM7QufG9pCtGhLXHz8/4Yt2BLLz1axrS88ow4/+SMDDYGy9Gd0dE53Zyp0hE1KKa3BOl0+ng7e19zUun08HV1RXBwcFYvHhxm3nGnKOpvzOPD6elhjgpFZg4pBO2PX8n/vfOLtC4KJGUUYjxH+3FE5/tw7GLxXKnSETUYprcE7V69Wr8/e9/xxNPPIEhQ4YAAPbt24fPP/8cCxcuRG5uLpYvXw61Wo0FCxZYPWGyLa4RRY3hqXHBi6N7YMqwEKyIP4W1+7OwPS0X29NyMbZfIObe0w0hvnx4NRE5tiYXUZ9//jneeustPPbYY9K2sWPHIjw8HB9++CHi4+PRqVMnvPbaayyi7FBmfhkAoJMPPwDp5vy9NHjtoXBMH9EZ/4o7iR8PXcRPhy5i85FLeGxwEGbd1RWBXPmciBxUk4fz9uzZgwEDBlyzfcCAAUhISAAADB8+HJmZmbeeHbU49kRRc4T4umPFxAH4+ZnhuKt7e9SaBb5OzMTIN7dhwfdHcP7yXDsiIkfS5CIqKCgIn3zyyTXbP/nkEwQFBQEA8vPz4e3tfevZUYtjEUW3onegFp9NHYJ1T0ZiWJd2qDHVFVN3Ld+O+d8dlhZyJSJyBE0ezlu+fDkeffRRbN68GYMHDwYAHDhwACdOnMC3334LANi/fz/Gjx9v3UzJ5sxmgazCCgBcaJNuzZBQH3w9fSj2pRfgnfiT2H06H9/sy8L6A+fx8G0dMfOurrx5gYjsnkI0Y5GX9PR0fPjhhzh58iQAoHv37njyyScREhJi7fzsWmOfAt1aXCquQOTSrXBSKpD2j9FwduIyYmQdB84V4J34U/j9VB6Aurv8Yvp3wN9GdkaYv6fM2RERWWrs53eziihqHHsrohLP5mP8R3vRyccNO1+8S+50yAElZRTinfhT2HnyykONo3r64W8ju2BQiI+MmRERXdHYz+8mD+cBQFFREfbt24ecnJxr1oOaPHlyc05JrUAGn5lHNjYw2Btf/HUIUrKKsGr7Gfxy3IDfUnPwW2oOBod4428ju+Cu7n5QKhVyp0pEdFNNLqJ++uknTJo0CaWlpfDy8oJCceUfO4VCwSLKjtVP+g3ipHKysf5BOqx6fCDO5JbiPzvP4ruDF7D/XCH2nzuAbv4eePKOLhjbLxAqZw4pE1Hr1eR/oebOnYu//vWvKC0tRVFREQoLC6VXQUGBLXKkFsI786ildWnvgdcf7ovf592FJ+/oDA+1M05ml2Lu+kMY/sZWvBt/CvmlVXKnSUTUoCYXURcuXMAzzzwDNzd+0Dqa+ke+BLOIohbm76XB/Pt7YvdLd+PF0d3h56lGTkkV3oo7icjXt+LFbw8h9VLDz+0kIpJLk4uo6OhoHDhwwBa5kMw4nEdy07q64H/v7Ipd8+7GOxP6o29HLaprzVh34Dzue+d3/Pk/e/Hb8WyYzbwfhojk1+Q5UWPGjMELL7yA48ePIzw8HC4uLhb7H3zwQaslRy2ntKoW+WXVAPjwYZKfylmJcf074MF+gTiYWYhPd53DlmMG7DmTjz1n8hHczg1/iQjGIwM7wttdJXe6RNRGNXmJA6Xy+p1XCoUCJpPplpNyFPa0xMHxi0bcv+J3eLu5IDn2XrnTIbrGhaIKfJFwDt8kZsJYWQugrth6IDwAk4YG47ZOOosbXYiImstmSxz8cUkDcgycVE6tXQedK+bf1xPPjgrDDykX8eXeDBy7aMR3yRfwXfIF9AzwwqSITogZ0AEe6mat3kJE1CT8l4YAAJkFZQCATu3cZc6E6MbcVM6YOKQTJgwOwqHzxfhybwZ+OnQRqZeMWLjhKJZuSkXMgA74y9Bg9Axo3T3ARGTfGlVErVixAjNmzIBGo8GKFStuGPvMM89YJTFqWVd6olxlzoSocRQKBfoH6dA/SIeFY3rivwcv4KvEDJzNLcNXiZn4KjET/Tpq8cigIDzYLxBaV5ebn5SIqAkaNScqNDQUBw4cQLt27RAaGnr9kykUOHv2rFUTtGf2NCdq8qf7sPNkLt54OBzjB3eSOx2iZhFCIOFMPr5KzMQvxwyovXwXn9pZidF99Hh0YBCGdWnHFdGJ6IasOicqPT29wT+T48jMvzyc58PhPLJfCoUCw7r6YlhXX+SVVmFD8gWsP3Aeadkl+CHlIn5IuYgOOlc8PLAjHh3Ykct5ENEt4QOIbcheeqJMZoHuCzej1iyw+6W70UHHIT1yHEIIHLlQjHUHsvBDykWUXL6zDwAiO7fDo4M6Irq3Hu6cjE5El9ns7jyTyYTVq1cjPj6+wQcQb926tenZkqwuFVeg1izg4qSA3ksjdzpEVqVQKNC3ow59O+qwcEwv/HLMgG+TzmPX6TwknM1Hwtl8uLocxT29/BEzIBAjwtrDxYnP7COim2tyEfXss89i9erVGDNmDPr06cN1WRxA5uXHvQR5u8GJc0XIgWlcnDCufweM698BF4oq8N+k8/g++QLS88rw46GL+PHQRfi4qzAmPAAxAwJxWydv/htHRNfV5CJqzZo1WLduHe6//35b5EMyyOTjXqgN6qBzxTOjwvD03V1x+HwxNqRcwE+HLiKvtBr/tzcD/7c3A5183DCufyDG9e+Arn4ecqdMRK1Mk4solUqFrl272iIXkgkX2qS2TKFQoF+QDv2CdPj7/T2x+0w+fki+gF+OGZBZUI53t57Gu1tPo4feE2PCA3B/3wB0ac+CioiaUUTNnTsX77zzDt577z12czuIjMtFVDCfmUdtnLOTEiO7tcfIbu1RUW1CXGo2fki+gB0nc3HCUIIThhK8FXeSBRURAWhGEbVr1y5s27YNmzdvRu/eva95APF3331nteSoZWRxOI/oGq4qJzzYLxAP9gtEUXk1fj2WjZ+PXMLu03nXFFT3hwfg/vAADvkRtTFNLqJ0Oh0eeughW+RCMslkTxTRDencVHhscBAeGxxUV1Adz8bPhy0Lqn9dLqju6xOAe3v7o4fek731RA6uSUVUbW0t7rrrLtx7773Q6/W2yolaUHFFDYrKawDU3Z1HRDemc1PhsUFBeGzQlYJq05FL2HXqSkH1799OIsjHFff01OPe3v4YFOwNZy6bQORwmvS32tnZGX/7299QVVVltQRWrlyJkJAQaDQaREREYN++fTeMX79+PXr06AGNRoPw8HBs2rTJYr8QArGxsQgICICrqyuioqJw6tQpi5iCggJMmjQJXl5e0Ol0mDZtGkpLS685z/Lly9GtWzeo1Wp06NABr732mnUa3YrUD+X5eqi42CBRE9UXVKunDsGBhVFY9khfRPX0h9pZiayCCny6Ox0TPtqLwa/9hrnrDmHLUQPKq2tvfmIisgtN/q/RkCFDkJycbJWLr127FnPmzMHixYtx8OBB9OvXD9HR0cjJyWkwfs+ePZg4cSKmTZuG5ORkxMTEICYmBkePHpVili1bhhUrVmDVqlVITEyEu7s7oqOjUVlZKcVMmjQJx44dQ1xcHDZu3IidO3dixowZFtd69tln8fHHH2P58uU4ceIEfvzxRwwZMsQq7W5NeGcekXXUF1QfTxmE5Nh7sOovA/HwbR2hc3NBYXkN/nvwPP72ZRIGvBqH//n8ANbtz0JuifX+Q0pELa/Jj31Zt24d5s+fj+eeew4DBw6Eu7vls9b69u3b6HNFRERg8ODBeO+99wAAZrMZQUFBePrpp/HSSy9dEz9+/HiUlZVh48aN0rahQ4eif//+WLVqFYQQCAwMxNy5c/H8888DAIqLi+Hv74/Vq1djwoQJSE1NRa9evbB//34MGjQIALBlyxbcf//9OH/+PAIDA5Gamoq+ffvi6NGj6N69e1N+PBbs4bEvH2w/gze2nEBM/0C8PWGA3OkQOZxakxkHMgrx67Fs/HrcgPOFFRb7+3XU4s7ufrirhx/6dtDy4chErYDNHvsyYcIEAMAzzzwjbVMoFBBCQKFQwGQyNeo81dXVSEpKwvz586VtSqUSUVFRSEhIaPCYhIQEzJkzx2JbdHQ0NmzYAKDu4cgGgwFRUVHSfq1Wi4iICCQkJGDChAlISEiATqeTCigAiIqKglKpRGJiIh566CH89NNP6Ny5MzZu3IjRo0dDCIGoqCgsW7YMPj4+121TVVWVxVCn0Whs1M9CTuyJIrItZyclhnZuh6Gd22HRAz1xwlCCX49lIy7VgKMXjDh0vhiHzhfjnfhTaOeuwshu7XFnDz/cEeYLnZtK7vSJ6AaaXESlp6db5cJ5eXkwmUzw9/e32O7v748TJ040eIzBYGgw3mAwSPvrt90oxs/Pz2K/s7MzfHx8pJizZ88iIyMD69evxxdffAGTyYTnnnsOjzzyyA2fDbh06VK88sorN2t6q5JZUAYA6NTO/SaRRHSrFAoFegZ4oWeAF56NCkOOsRLbT+Zie1oOfj+Zh/yyanyXfAHfJV+AUgHc1skbd/Xww53d26NXgBfv9iNqZZpcRAUHB9sij1bFbDajqqoKX3zxBbp16wYA+OSTTzBw4ECkpaVdd4hv/vz5Fj1lRqMRQUFBLZJzc7Enikg+fl4a6U6/GpMZSRmF2JaWg20ncnAyuxQHMgpxIKMQb/6SBn8vNUaEtceIMF/c3tUXvh5qudMnavOafTvW8ePHkZmZierqaovtDz74YKOO9/X1hZOTE7Kzsy22Z2dnX3f5BL1ef8P4+q/Z2dkICAiwiOnfv78U88eJ67W1tSgoKJCODwgIgLOzs1RAAUDPnj0BAJmZmdctotRqNdRq+/mHrcZkxsWiugn3LKKI5OVy1bDf/Pt64nxhOban1fVS7T6dj2xjFb5NOo9vk84DAHoFeGFEmC+Gh/licIgPNC5OMreAqO1pchF19uxZPPTQQzhy5Ig0FwqA1M3c2DlRKpUKAwcORHx8PGJiYgDU9QDFx8dj1qxZDR4TGRmJ+Ph4zJ49W9oWFxeHyMhIAEBoaCj0ej3i4+OlosloNCIxMRFPPfWUdI6ioiIkJSVh4MCBAICtW7fCbDYjIiICAHD77bejtrYWZ86cQZcuXQAAJ0+eBOBYPXEXiypgMguonZXw87Sf4o+oLejo7Ya/DA3GX4YGo7LGhH3pBdh1Og+/n8pD6iUjjl9+fbjzLFTOSgwJ8cHwMF8M7+qLXgFenKBO1AKaXEQ9++yzCA0NRXx8PEJDQ7Fv3z7k5+dj7ty5WL58eZPONWfOHEyZMgWDBg3CkCFD8Pbbb6OsrAxTp04FAEyePBkdOnTA0qVLpWuPHDkSb731FsaMGYM1a9bgwIED+OijjwDUFXKzZ8/GkiVLEBYWhtDQUCxatAiBgYFSodazZ0+MHj0a06dPx6pVq1BTU4NZs2ZhwoQJCAwMBFA30fy2227DX//6V7z99tswm82YOXMm7rnnHoveKXuXedXjXvgPLlHrpXFxwh3d2uOObu0BALklVdhzpq6g2nUqDwZjJXadzsOu03kAgHbuKgzr6ovbu9T1bAW3c+N8KiIbaHIRlZCQgK1bt8LX1xdKpRJKpRLDhw/H0qVL8cwzzzRpDanx48cjNzcXsbGxMBgM6N+/P7Zs2SJNDM/MzIRSeWUpq2HDhuHrr7/GwoULsWDBAoSFhWHDhg3o06ePFPPiiy+irKwMM2bMQFFREYYPH44tW7ZAo9FIMV999RVmzZqFUaNGQalU4uGHH8aKFSuk/UqlEj/99BOefvpp3HHHHXB3d8d9992Ht956q6k/rlaN86GI7FN7TzXG9e+Acf07QAiBM7mlUkG192w+8suq8dOhi/jp0EUAQIBWc3mo0AeRnX0R5OPKoorICpq8TpS3tzcOHjyI0NBQdOnSBR9//DHuuusunDlzBuHh4SgvL7dVrnanta8TtXRTKj7ceRZPDAvByw/2ljsdIrKC6lozUrKKsOtULvaeLUByViFqTJb/zAdqNRh6uZcqsnM7Pnyc6A9stk5Unz59cOjQIYSGhiIiIgLLli2DSqXCRx99hM6dO99S0tSy2BNF5HhUzkoMCfXBkNC6Ne0qqk04mFmIhDP52Hs2HylZRbhYXInvDl7AdwcvAAA66FylnqohoT7o5MPhP6LGaHIRtXDhQpSV1a0t9Oqrr+KBBx7AiBEj0K5dO6xdu9bqCZLt1BdRwe1YRBE5KleVE27vWrcsAgCUV9ciKeNKUXX4fDEuFFXgvwfP478H6+78a++pxuAQbwwK9sHgEB/0DPDkA5SJGtDk4byGFBQUwNvbm/9z+YPWPJwnhEDfl39FSVUt4p67A2H+nnKnREQyKKuqxYGMQuw9W1dUHb1QfM3wn5vKCbd18sagEG8MDvFB/yAdH1hODs1mw3n1Tp8+jTNnzuCOO+6Aj48PrFCLUQsqKq9BSVXd0+Q5H4Ko7XJXO2Nkt/YYefnOv8oaEw5lFdUt9HmuAAcyClFSWWtx95+TUoHegV4YFOyDQSF1xZWfp+ZGlyFySE0uovLz8/HYY49h27ZtUCgUOHXqFDp37oxp06bB29vb4e5gc1T1Q3n+Xmou0kdEEo2LEyI6t0NE53YAALNZ4GROCfafu1xUnSvEhaIKHD5fjMPni/Hp7rpHgQX5uKJ/kDcGBOkwoJMOvQK9oHbmvy3k2JpcRD333HNwcXFBZmamtIo3ULdcwZw5c1hE2YkMTionokZQKhXoofdCD70XHh9at9jwxaIKqadq/7lCnDAYkVVQgayCCmlZBZWTEr0CvTCgkw4DOtUVVx29ubQCOZYmF1G//vorfvnlF3Ts2NFie1hYGDIyMqyWGNlW1lULbRIRNUWgzhUP6lzxYL+6BYpLKmtw+HwxkjMLkZxZhOSsIhSUVSMlqwgpWUX4bPc5AICvh6qut6qTDgOCdOgbpIMH51aRHWvyb29ZWRnc3K794C0oKLCr58a1dRn5dXdYBvu4y5wJEdk7T42LxR2AQghkFVQgOetyUZVZiOOXjMgrrcZvqdn4LbXuGagKBdC1vQfCO2rRt4MW4R116BXgBVcVhwHJPjS5iBoxYgS++OIL/OMf/wBQ96gVs9mMZcuW4a677rJ6gmQb0hpR7VxlzoSIHI1CoUCndm7o1M4N4/p3AFA3Yf3YRWNdb1VWEVIyi3ChqAKnckpxKqdUWrPKSalAmJ8Hwjto0bdjXWHVQ+/JuZvUKjW5iFq2bBlGjRqFAwcOoLq6Gi+++CKOHTuGgoIC7N692xY5kg1kFVQA4JwoImoZGhcnDAz2xsBgb2lbTkkljl6om6B+5HwxDp0vRl5pFU4YSnDCUIL1SXXrVjkrFeiu96wrqjro0LejFt38PaFy5tpVJK9mrVh+8uRJvPfee/D09ERpaSn+9Kc/YebMmQgICLBFjmRlVbUmXCyuL6I4nEdE8vDz1ODuHhrc3aPuealCCGQbq3D4fFFdcXW5wCooq8axi0Ycu2jEN8gCUDdxvbveE70DvdAr0Au9ArzQI8CLc6yoRTXrt02r1eLvf/+7xbbz589jxowZ+Oijj6ySGNnOhcIKCAG4ujjB10MldzpERADqhgH1Wg30Wj3u7a0HUFdYXSyuxJHzRXU9VpcLq+KKGhy5UPf91ULauaFXoBd6B2rRK6CuwPLzVPOuQLIJq5Xs+fn5+OSTT1hE2YGrn5nHf1iIqDVTKBTooHNFB50rRvepG+2on7h+7GIxjl+q66E6ftEIg7ES5/LLcS6/HJuOGKRztHNXSb1V9V9Dfd35KBu6Zez3bIOuTCrnfCgisj9XT1y/L/zKNJL80iqkXirB8UvFOH7RiOOXjDidU4r8smr8fioPv5/Kk2LVzkr00Huih94L3fSe6KH3RHe9J3w9eJc5NR6LqDYoM58LbRKR42nnocbwMDWGh/lK2yprTEgzlOD4JaNUWKVeMqK82oRDlyezX83XQ4Xuek909/dCd70Huuu90M3fA24qflzStfhb0QbV90QFsyeKiBycxsUJ/YJ06Bekk7aZzQIZBeU4ftGINIMRJwwlSMsuQWZBOfJKq5F3Oh+7T+dL8QpF3X86u/vX9VjV91yFtOOQYFvX6CLqT3/60w33FxUV3Wou1EIyuVo5EbVhSqUCob7uCPV1x5i+V4YDy6trcSq7FGmXl1hIyzYizVCCvNJqZOSXIyO/HL8ez5biVU5KhPq6o6u/B8L8PBDm54kwfw+EtHPn8gttRKOLKK1We9P9kydPvuWEyLaEEBYTy4mIqI6byvmaXisAyCutwsn6wspQghPZJThpKEFFjQlp2XW9WFdzUioQ3M7NorDq6ueBLu09uGiog2l0EfXZZ5/ZMg9qIfll1SivNkGhADp6c7VyIqKb8fVQw7erGsO6XplrZTYLXCiqwOmcUpzKKcGp7LqV10/nlKK0qhZnc8twNrcMvxy70nNVPyzYtb3H5d4rT3T180Dn9u7w0rjI0TS6RZwT1cZkXJ5UHuClgdqZ/yMiImoOpVKBIB83BPm44a4eftL2+gVDLQurEpzKKUVReY00LBh/IsfifL4eanRu744u7d3R2beusOrc3gNB3q6cd9WKsYhqY7I4H4qIyGauLBiqwYiw9tJ2IQTyy6pxKvtKUXUyuwRncsuQW1KFvNK61770AovzuTgp0MnHDZ3b1xVWXa4qsHzcuViy3FhEtTH1PVG8M4+IqOUoFIq6YUEPNSK7tLPYZ6ysQXpuGc7mlUrDgGfzypCeV4rKGjPO5JbhTG7ZNefUubmgs6+7VGCFtnNHiK87gtu5cUmGFsKfchvDSeVERK2Ll8alwQntZrPAJWMlzubWF1elOJtXV2RdKKpAUXkNDmYW4WBm0TXn9PNUI6RdXUEV4usu/Tm4nRs8Of/KalhEtTEcziMisg9K5ZVH3lw9NAgAFdUmpOdd3XtVinP55cjIL0NheQ1ySqqQU1KFfecKrjmvr4fqclHljpB2bgj2vfy1nTu0riywmoJFVBuTUVDXJRzczl3mTIiIqLlcVU51zwEM9LpmX3F5DTIKyuqeI5hXhnP5ZZcntJfVLSZ6+XUgo/CaY33cVXW9V+3cEeTtKk2e7+TjBn8vDZyUfN7q1VhEtSGVNSZkG6sAcDiPiMhRad1c0NdNh74dddfsK6msu0OwvrCqL7LO5Zcjt6QKBWXVKCirRnIDQ4QuTgp09HZDR29XdLqquAryrvuqdWt7vVgsotqQ84V1Q3keamd4t8FfdiKits5T44I+HbTo0+HaBbTLqmqlHqtz+eXIKixHVkHd63xhBWpMAul5ZUjPu3aSe925na8UVe3cLHqyOnq7OuSyOiyi2pCMqx48rFCwS5aIiK5wVztfd4jQZBYwGCuR+YfiKrOgHFmFFcgtqUJJZS2OXTTi2EXjNccrFIC/pwadLhdUHbzr5nrVfw3Uudrlau4sotoQ3plHRETN4XTVJPdItLtmf0W1CecLLxdVBeXILKhAZkG5tK282gSDsRIGYyX2nWv4Gu091VJh1bH+q7crOujc0MHbFR7q1leytL6MyGakIoprRBERkRW5qpwQ5u+JMH/Pa/YJIVBQVi31Wp0vLMeFwgpcKKqQvpZXm5BbUoXckiqkZBU1eA2tqws66FwterI6ervizu5+svVisYhqQzLz2RNFREQtS6FQoJ2HGu081BjQyfua/UIIFJXX4HxhBS4UlV/+eqXAOl9YgeKKGul1/JLlcOHRV6JbqinXYBHVhnA4j4iIWhuFQgFvdxW83VUI73jthHcAKK2qvVxU1fVina8vrsprZB3mYxHVRgghpCKKj3whIiJ74qF2Rne9J7rrrx0ulBMfDd1G5JRUoarWDKUCCNS5yp0OERGR3WMR1UbU90IF6lzh4sS3nYiI6Fbx07SNqF8jikN5RERE1sEiqo3gpHIiIiLrahVF1MqVKxESEgKNRoOIiAjs27fvhvHr169Hjx49oNFoEB4ejk2bNlnsF0IgNjYWAQEBcHV1RVRUFE6dOmURU1BQgEmTJsHLyws6nQ7Tpk1DaWlpg9c7ffo0PD09odPpbqmdcsq6XEQFsYgiIiKyCtmLqLVr12LOnDlYvHgxDh48iH79+iE6Oho5OTkNxu/ZswcTJ07EtGnTkJycjJiYGMTExODo0aNSzLJly7BixQqsWrUKiYmJcHd3R3R0NCorK6WYSZMm4dixY4iLi8PGjRuxc+dOzJgx45rr1dTUYOLEiRgxYoT1G9+CMvLrnnUU7OMucyZERESOQSGEEHImEBERgcGDB+O9994DAJjNZgQFBeHpp5/GSy+9dE38+PHjUVZWho0bN0rbhg4div79+2PVqlUQQiAwMBBz587F888/DwAoLi6Gv78/Vq9ejQkTJiA1NRW9evXC/v37MWjQIADAli1bcP/99+P8+fMIDAyUzj1v3jxcvHgRo0aNwuzZs1FUVNTothmNRmi1WhQXF8PL69pnEbWkQUt+Q15pFX6aNfy663AQERFR4z+/Ze2Jqq6uRlJSEqKioqRtSqUSUVFRSEhIaPCYhIQEi3gAiI6OluLT09NhMBgsYrRaLSIiIqSYhIQE6HQ6qYACgKioKCiVSiQmJkrbtm7divXr12PlypWNak9VVRWMRqPFqzUor65FXmkVAM6JIiIishZZi6i8vDyYTCb4+/tbbPf394fBYGjwGIPBcMP4+q83i/Hz87PY7+zsDB8fHykmPz8fTzzxBFavXt3oXqSlS5dCq9VKr6CgoEYdZ2v1k8q1ri7QurnInA0REZFjkH1OVGs1ffp0/PnPf8Ydd9zR6GPmz5+P4uJi6ZWVlWXDDBuPz8wjIiKyPlmLKF9fXzg5OSE7O9tie3Z2NvR6fYPH6PX6G8bXf71ZzB8nrtfW1qKgoECK2bp1K5YvXw5nZ2c4Oztj2rRpKC4uhrOzMz799NMGc1Or1fDy8rJ4tQZc3oCIiMj6ZC2iVCoVBg4ciPj4eGmb2WxGfHw8IiMjGzwmMjLSIh4A4uLipPjQ0FDo9XqLGKPRiMTERCkmMjISRUVFSEpKkmK2bt0Ks9mMiIgIAHXzplJSUqTXq6++Ck9PT6SkpOChhx6yzg+ghUhFFBfaJCIishrZH0A8Z84cTJkyBYMGDcKQIUPw9ttvo6ysDFOnTgUATJ48GR06dMDSpUsBAM8++yxGjhyJt956C2PGjMGaNWtw4MABfPTRRwDqngY9e/ZsLFmyBGFhYQgNDcWiRYsQGBiImJgYAEDPnj0xevRoTJ8+HatWrUJNTQ1mzZqFCRMmSHfm9ezZ0yLPAwcOQKlUok+fPi30k7Ee9kQRERFZn+xF1Pjx45Gbm4vY2FgYDAb0798fW7ZskSaGZ2ZmQqm80mE2bNgwfP3111i4cCEWLFiAsLAwbNiwwaK4efHFF1FWVoYZM2agqKgIw4cPx5YtW6DRaKSYr776CrNmzcKoUaOgVCrx8MMPY8WKFS3X8BZUX0QFs4giIiKyGtnXiXJkrWGdKJNZoOeiLag2mfH7i3dxxXIiIqKbsIt1osj2so2VqDaZ4axUIECrufkBRERE1Cgsohxc/VBeR29XODvx7SYiIrIWfqo6uPo1ojiMR0REZF0sohwc78wjIiKyDRZRDi6j/s48rhFFRERkVSyiHBx7ooiIiGyDRZSDyyrgnCgiIiJbYBHlwEoqa1BQVg2APVFERETWxiLKgdUP5fm4q+CpcZE5GyIiIsfCIsqBcSiPiIjIdlhEObCMfD4zj4iIyFZYRDkw3plHRERkOyyiHJhURHGNKCIiIqtjEeXA2BNFRERkOyyiHFStyYwLhRUAWEQRERHZAosoB3WpuBK1ZgGVkxJ6L43c6RARETkcFlEOqn4or6OPK5RKhczZEBEROR4WUQ6K86GIiIhsi0WUg+IaUURERLbFIspBcbVyIiIi22IR5aA4nEdERGRbLKIcVEZ+GQAguJ27zJkQERE5JhZRDqi4vAbGyloAQJCPq8zZEBEROSYWUQ6ofijP10MNN5WzzNkQERE5JhZRDiijoH4oj/OhiIiIbIVFlAPipHIiIiLbYxHlgLJYRBEREdkciygHVL/QJosoIiIi22ER5YCk4TzOiSIiIrIZFlEOpsZkxsWiCgB85AsREZEtsYhyMBcKK2AWgNpZifaearnTISIiclgsohzM1XfmKRQKmbMhIiJyXCyiHEzG5SKKa0QRERHZFosoB1O/vEEQ50MRERHZFIsoB5PJ5Q2IiIhaBIsoB8PhPCIiopbBIsqBCCG4WjkREVELaRVF1MqVKxESEgKNRoOIiAjs27fvhvHr169Hjx49oNFoEB4ejk2bNlnsF0IgNjYWAQEBcHV1RVRUFE6dOmURU1BQgEmTJsHLyws6nQ7Tpk1DaWmptH/79u0YN24cAgIC4O7ujv79++Orr76yXqNtoLC8BqVVtQCAjt4sooiIiGxJ9iJq7dq1mDNnDhYvXoyDBw+iX79+iI6ORk5OToPxe/bswcSJEzFt2jQkJycjJiYGMTExOHr0qBSzbNkyrFixAqtWrUJiYiLc3d0RHR2NyspKKWbSpEk4duwY4uLisHHjRuzcuRMzZsywuE7fvn3x3//+F4cPH8bUqVMxefJkbNy40XY/jFuUkV8GANB7aaBxcZI5GyIiIgcnZDZkyBAxc+ZM6XuTySQCAwPF0qVLG4x/7LHHxJgxYyy2RUREiCeffFIIIYTZbBZ6vV68+eab0v6ioiKhVqvFN998I4QQ4vjx4wKA2L9/vxSzefNmoVAoxIULF66b6/333y+mTp3a6LYVFxcLAKK4uLjRx9yKDcnnRfC8jeLRD/a0yPWIiIgcUWM/v2XtiaqurkZSUhKioqKkbUqlElFRUUhISGjwmISEBIt4AIiOjpbi09PTYTAYLGK0Wi0iIiKkmISEBOh0OgwaNEiKiYqKglKpRGJi4nXzLS4uho+Pz3X3V1VVwWg0WrxaEpc3ICIiajmyFlF5eXkwmUzw9/e32O7v7w+DwdDgMQaD4Ybx9V9vFuPn52ex39nZGT4+Pte97rp167B//35MnTr1uu1ZunQptFqt9AoKCrpurC1k5PPOPCIiopYi+5woe7Bt2zZMnToV//nPf9C7d+/rxs2fPx/FxcXSKysrqwWztHzkCxEREdmWrEWUr68vnJyckJ2dbbE9Ozsber2+wWP0ev0N4+u/3izmjxPXa2trUVBQcM11d+zYgbFjx+Lf//43Jk+efMP2qNVqeHl5WbxakrS8AXuiiIiIbE7WIkqlUmHgwIGIj4+XtpnNZsTHxyMyMrLBYyIjIy3iASAuLk6KDw0NhV6vt4gxGo1ITEyUYiIjI1FUVISkpCQpZuvWrTCbzYiIiJC2bd++HWPGjMEbb7xhcedea1RVa8IlY93dh+yJIiIisj1nuROYM2cOpkyZgkGDBmHIkCF4++23UVZWJs09mjx5Mjp06IClS5cCAJ599lmMHDkSb731FsaMGYM1a9bgwIED+OijjwAACoUCs2fPxpIlSxAWFobQ0FAsWrQIgYGBiImJAQD07NkTo0ePxvTp07Fq1SrU1NRg1qxZmDBhAgIDAwHUDeE98MADePbZZ/Hwww9Lc6VUKtUNJ5fL5XxhBYQA3FROaOeukjsdIiIix9dCdwve0Lvvvis6deokVCqVGDJkiNi7d6+0b+TIkWLKlCkW8evWrRPdunUTKpVK9O7dW/z8888W+81ms1i0aJHw9/cXarVajBo1SqSlpVnE5Ofni4kTJwoPDw/h5eUlpk6dKkpKSqT9U6ZMEQCueY0cObLR7WrJJQ62pmaL4HkbRfS/d9j8WkRERI6ssZ/fCiGEkLGGc2hGoxFarRbFxcU2nx/1+Z5zWPzjMdzbyx8fTR508wOIiIioQY39/ObdeQ6Cd+YRERG1LBZRDoJrRBEREbUsFlEOgquVExERtSwWUQ5ACMHhPCIiohbGIsoB5JZWoaLGBIUC6OjNIoqIiKglsIhyAPVDeYFaV6ic+ZYSERG1BH7iOoBMaT6Uq8yZEBERtR0sohyAdGeej7vMmRAREbUdLKIcQCYfPExERNTiWEQ5gCzemUdERNTiWEQ5gPrhPBZRRERELYdFlJ2rqDYhp6QKAIsoIiKilsQiys6dL6zrhfLUOEPn5iJzNkRERG0Hiyg7d/VQnkKhkDkbIiKitoNFlJ3j416IiIjkwSLKznF5AyIiInmwiLJz7IkiIiKSB4soO8ciioiISB4souyY2SykIoqPfCEiImpZLKLsWE5JFaprzXBSKhCg08idDhERUZvCIsqO1fdCBeo0cHHiW0lERNSS+MlrxzLyywBwKI+IiEgOLKLsWP2Dh4M4qZyIiKjFsYiyY9Kkcq4RRURE1OJYRNmxDC5vQEREJBsWUXYsi0UUERGRbFhE2amyqlrklVYD4CNfiIiI5MAiyk7Vz4fSubnAS+MiczZERERtD4soO8XHvRAREcmLRZSdysxnEUVERCQnFlF2ij1RRERE8mIRZadYRBEREcmLRZSdkooo3plHREQkCxZRdshkFjhfyJ4oIiIiObGIskMGYyVqTAIuTgoEaF3lToeIiKhNYhFlhzLyywAAHb3d4KRUyJwNERFR28Qiyg7VP+4liEN5REREsmkVRdTKlSsREhICjUaDiIgI7Nu374bx69evR48ePaDRaBAeHo5NmzZZ7BdCIDY2FgEBAXB1dUVUVBROnTplEVNQUIBJkybBy8sLOp0O06ZNQ2lpqUXM4cOHMWLECGg0GgQFBWHZsmXWafAtunJnHofyiIiI5CJ7EbV27VrMmTMHixcvxsGDB9GvXz9ER0cjJyenwfg9e/Zg4sSJmDZtGpKTkxETE4OYmBgcPXpUilm2bBlWrFiBVatWITExEe7u7oiOjkZlZaUUM2nSJBw7dgxxcXHYuHEjdu7ciRkzZkj7jUYj7r33XgQHByMpKQlvvvkmXn75ZXz00Ue2+2E0UsblhTaDfdxlzoSIiKgNEzIbMmSImDlzpvS9yWQSgYGBYunSpQ3GP/bYY2LMmDEW2yIiIsSTTz4phBDCbDYLvV4v3nzzTWl/UVGRUKvV4ptvvhFCCHH8+HEBQOzfv1+K2bx5s1AoFOLChQtCCCHef/994e3tLaqqqqSYefPmie7duze6bcXFxQKAKC4ubvQxjfHgu7+L4HkbxeYjl6x6XiIiImr857esPVHV1dVISkpCVFSUtE2pVCIqKgoJCQkNHpOQkGARDwDR0dFSfHp6OgwGg0WMVqtFRESEFJOQkACdTodBgwZJMVFRUVAqlUhMTJRi7rjjDqhUKovrpKWlobCwsMHcqqqqYDQaLV62UD+cF8w1ooiIiGQjaxGVl5cHk8kEf39/i+3+/v4wGAwNHmMwGG4YX//1ZjF+fn4W+52dneHj42MR09A5rr7GHy1duhRarVZ6BQUFNdzwW1BRbYLKue5t48RyIiIi+cg+J8qRzJ8/H8XFxdIrKyvL6tdwVTkhcUEUTvxjNDzUzlY/PxERETWOrEWUr68vnJyckJ2dbbE9Ozsber2+wWP0ev0N4+u/3izmjxPXa2trUVBQYBHT0DmuvsYfqdVqeHl5WbxsRePiZLNzExER0c3JWkSpVCoMHDgQ8fHx0jaz2Yz4+HhERkY2eExkZKRFPADExcVJ8aGhodDr9RYxRqMRiYmJUkxkZCSKioqQlJQkxWzduhVmsxkRERFSzM6dO1FTU2Nxne7du8Pb2/sWW05ERER2r4Umul/XmjVrhFqtFqtXrxbHjx8XM2bMEDqdThgMBiGEEI8//rh46aWXpPjdu3cLZ2dnsXz5cpGamioWL14sXFxcxJEjR6SY119/Xeh0OvHDDz+Iw4cPi3HjxonQ0FBRUVEhxYwePVoMGDBAJCYmil27domwsDAxceJEaX9RUZHw9/cXjz/+uDh69KhYs2aNcHNzEx9++GGj22aru/OIiIjIdhr7+S17ESWEEO+++67o1KmTUKlUYsiQIWLv3r3SvpEjR4opU6ZYxK9bt05069ZNqFQq0bt3b/Hzzz9b7DebzWLRokXC399fqNVqMWrUKJGWlmYRk5+fLyZOnCg8PDyEl5eXmDp1qigpKbGIOXTokBg+fLhQq9WiQ4cO4vXXX29Su1hEERER2Z/Gfn4rhBBC3r4wx2U0GqHValFcXGzT+VFERERkPY39/ObdeURERETNwCKKiIiIqBlYRBERERE1A4soIiIiomZgEUVERETUDCyiiIiIiJqBRRQRERFRM7CIIiIiImoGFlFEREREzeAsdwKOrH4xeKPRKHMmRERE1Fj1n9s3e6gLiygbKikpAQAEBQXJnAkRERE1VUlJCbRa7XX389l5NmQ2m3Hx4kV4enpCoVBY7bxGoxFBQUHIyspyyGfyOXr7AMdvo6O3D3D8NrJ99s/R22jL9gkhUFJSgsDAQCiV15/5xJ4oG1IqlejYsaPNzu/l5eWQfzHqOXr7AMdvo6O3D3D8NrJ99s/R22ir9t2oB6oeJ5YTERERNQOLKCIiIqJmYBFlh9RqNRYvXgy1Wi13Kjbh6O0DHL+Njt4+wPHbyPbZP0dvY2toHyeWExERETUDe6KIiIiImoFFFBEREVEzsIgiIiIiagYWUURERETNwCLKDq1cuRIhISHQaDSIiIjAvn375E7pGi+//DIUCoXFq0ePHtL+yspKzJw5E+3atYOHhwcefvhhZGdnW5wjMzMTY8aMgZubG/z8/PDCCy+gtrbWImb79u247bbboFar0bVrV6xevdom7dm5cyfGjh2LwMBAKBQKbNiwwWK/EAKxsbEICAiAq6sroqKicOrUKYuYgoICTJo0CV5eXtDpdJg2bRpKS0stYg4fPowRI0ZAo9EgKCgIy5YtuyaX9evXo0ePHtBoNAgPD8emTZtapI1PPPHENe/p6NGj7aaNS5cuxeDBg+Hp6Qk/Pz/ExMQgLS3NIqYlfy+t/fe4Me278847r3kP//a3v9lF+z744AP07dtXWlgxMjISmzdvlvbb83vX2Dba8/vXkNdffx0KhQKzZ8+Wttnd+yjIrqxZs0aoVCrx6aefimPHjonp06cLnU4nsrOz5U7NwuLFi0Xv3r3FpUuXpFdubq60/29/+5sICgoS8fHx4sCBA2Lo0KFi2LBh0v7a2lrRp08fERUVJZKTk8WmTZuEr6+vmD9/vhRz9uxZ4ebmJubMmSOOHz8u3n33XeHk5CS2bNli9fZs2rRJ/P3vfxffffedACC+//57i/2vv/660Gq1YsOGDeLQoUPiwQcfFKGhoaKiokKKGT16tOjXr5/Yu3ev+P3330XXrl3FxIkTpf3FxcXC399fTJo0SRw9elR88803wtXVVXz44YdSzO7du4WTk5NYtmyZOH78uFi4cKFwcXERR44csXkbp0yZIkaPHm3xnhYUFFjEtOY2RkdHi88++0wcPXpUpKSkiPvvv1906tRJlJaWSjEt9Xtpi7/HjWnfyJEjxfTp0y3ew+LiYrto348//ih+/vlncfLkSZGWliYWLFggXFxcxNGjR4UQ9v3eNbaN9vz+/dG+fftESEiI6Nu3r3j22Wel7fb2PrKIsjNDhgwRM2fOlL43mUwiMDBQLF26VMasrrV48WLRr1+/BvcVFRUJFxcXsX79emlbamqqACASEhKEEHUf6EqlUhgMBinmgw8+EF5eXqKqqkoIIcSLL74oevfubXHu8ePHi+joaCu3xtIfCwyz2Sz0er148803pW1FRUVCrVaLb775RgghxPHjxwUAsX//film8+bNQqFQiAsXLgghhHj//feFt7e31D4hhJg3b57o3r279P1jjz0mxowZY5FPRESEePLJJ23aRiHqiqhx48Zd9xh7a2NOTo4AIHbs2CGEaNnfy5b4e/zH9glR9yF89QfWH9lT+4QQwtvbW3z88ccO99411EYhHOf9KykpEWFhYSIuLs6iTfb4PnI4z45UV1cjKSkJUVFR0jalUomoqCgkJCTImFnDTp06hcDAQHTu3BmTJk1CZmYmACApKQk1NTUW7ejRowc6deoktSMhIQHh4eHw9/eXYqKjo2E0GnHs2DEp5upz1Me09M8iPT0dBoPBIhetVouIiAiL9uh0OgwaNEiKiYqKglKpRGJiohRzxx13QKVSSTHR0dFIS0tDYWGhFCNnm7dv3w4/Pz90794dTz31FPLz86V99tbG4uJiAICPjw+Alvu9bKm/x39sX72vvvoKvr6+6NOnD+bPn4/y8nJpn720z2QyYc2aNSgrK0NkZKTDvXcNtbGeI7x/M2fOxJgxY67Jwx7fRz6A2I7k5eXBZDJZ/PIAgL+/P06cOCFTVg2LiIjA6tWr0b17d1y6dAmvvPIKRowYgaNHj8JgMEClUkGn01kc4+/vD4PBAAAwGAwNtrN+341ijEYjKioq4OrqaqPWWarPp6Fcrs7Vz8/PYr+zszN8fHwsYkJDQ685R/0+b2/v67a5/hy2NHr0aPzpT39CaGgozpw5gwULFuC+++5DQkICnJyc7KqNZrMZs2fPxu23344+ffpI12+J38vCwkKb/z1uqH0A8Oc//xnBwcEIDAzE4cOHMW/ePKSlpeG7776zi/YdOXIEkZGRqKyshIeHB77//nv06tULKSkpDvPeXa+NgP2/fwCwZs0aHDx4EPv3779mnz3+HWQRRTZx3333SX/u27cvIiIiEBwcjHXr1rVYcUPWNWHCBOnP4eHh6Nu3L7p06YLt27dj1KhRMmbWdDNnzsTRo0exa9cuuVOxieu1b8aMGdKfw8PDERAQgFGjRuHMmTPo0qVLS6fZZN27d0dKSgqKi4vx7bffYsqUKdixY4fcaVnV9drYq1cvu3//srKy8OyzzyIuLg4ajUbudKyCw3l2xNfXF05OTtfcqZCdnQ29Xi9TVo2j0+nQrVs3nD59Gnq9HtXV1SgqKrKIuboder2+wXbW77tRjJeXV4sWavX53Oh90ev1yMnJsdhfW1uLgoICq7RZjve/c+fO8PX1xenTp6Xc7KGNs2bNwsaNG7Ft2zZ07NhR2t5Sv5e2/nt8vfY1JCIiAgAs3sPW3D6VSoWuXbti4MCBWLp0Kfr164d33nnHYd67G7WxIfb2/iUlJSEnJwe33XYbnJ2d4ezsjB07dmDFihVwdnaGv7+/3b2PLKLsiEqlwsCBAxEfHy9tM5vNiI+Ptxgzb41KS0tx5swZBAQEYODAgXBxcbFoR1paGjIzM6V2REZG4siRIxYfynFxcfDy8pK6tiMjIy3OUR/T0j+L0NBQ6PV6i1yMRiMSExMt2lNUVISkpCQpZuvWrTCbzdI/hJGRkdi5cydqamqkmLi4OHTv3h3e3t5STGtoMwCcP38e+fn5CAgIkHJrzW0UQmDWrFn4/vvvsXXr1muGFVvq99JWf49v1r6GpKSkAIDFe9ha29cQs9mMqqoqu3/vGtPGhtjb+zdq1CgcOXIEKSkp0mvQoEGYNGmS9Ge7ex+bNA2dZLdmzRqhVqvF6tWrxfHjx8WMGTOETqezuFOhNZg7d67Yvn27SE9PF7t37xZRUVHC19dX5OTkCCHqbmPt1KmT2Lp1qzhw4ICIjIwUkZGR0vH1t7Hee++9IiUlRWzZskW0b9++wdtYX3jhBZGamipWrlxpsyUOSkpKRHJyskhOThYAxL/+9S+RnJwsMjIyhBB1SxzodDrxww8/iMOHD4tx48Y1uMTBgAEDRGJioti1a5cICwuzuP2/qKhI+Pv7i8cff1wcPXpUrFmzRri5uV1z+7+zs7NYvny5SE1NFYsXL7baEgc3amNJSYl4/vnnRUJCgkhPTxe//fabuO2220RYWJiorKy0izY+9dRTQqvViu3bt1vcIl5eXi7FtNTvpS3+Ht+sfadPnxavvvqqOHDggEhPTxc//PCD6Ny5s7jjjjvson0vvfSS2LFjh0hPTxeHDx8WL730klAoFOLXX38VQtj3e9eYNtr7+3c9f7zj0N7eRxZRdujdd98VnTp1EiqVSgwZMkTs3btX7pSuMX78eBEQECBUKpXo0KGDGD9+vDh9+rS0v6KiQvzv//6v8Pb2Fm5ubuKhhx4Sly5dsjjHuXPnxH333SdcXV2Fr6+vmDt3rqipqbGI2bZtm+jfv79QqVSic+fO4rPPPrNJe7Zt2yYAXPOaMmWKEKJumYNFixYJf39/oVarxahRo0RaWprFOfLz88XEiROFh4eH8PLyElOnThUlJSUWMYcOHRLDhw8XarVadOjQQbz++uvX5LJu3TrRrVs3oVKpRO/evcXPP/9s8zaWl5eLe++9V7Rv3164uLiI4OBgMX369Gv+wWnNbWyobQAsfmda8vfS2n+Pb9a+zMxMcccddwgfHx+hVqtF165dxQsvvGCxzlBrbt9f//pXERwcLFQqlWjfvr0YNWqUVEAJYd/vXWPaaO/v3/X8sYiyt/dRIYQQTeu7IiIiIiLOiSIiIiJqBhZRRERERM3AIoqIiIioGVhEERERETUDiygiIiKiZmARRURERNQMLKKIiIiImoFFFBEREVEzsIgiIgIQEhKCt99+W+40iMiOsIgiIruiUChu+Hr55Zebdd79+/djxowZt5Rbeno6/vznPyMwMBAajQYdO3bEuHHjcOLECQDAuXPnoFAopAfHEpF9c5Y7ASKiprh06ZL057Vr1yI2NhZpaWnSNg8PD+nPQgiYTCY4O9/8n7r27dvfUl41NTW455570L17d3z33XcICAjA+fPnsXnzZhQVFd3SuYmodWJPFBHZFb1eL720Wi0UCoX0/YkTJ+Dp6YnNmzdj4MCBUKvV2LVrF86cOYNx48bB398fHh4eGDx4MH777TeL8/5xOE+hUODjjz/GQw89BDc3N4SFheHHH3+8bl7Hjh3DmTNn8P7772Po0KEIDg7G7bffjiVLlmDo0KEAgNDQUADAgAEDoFAocOedd0rHf/zxx+jZsyc0Gg169OiB999/X9pX34O1Zs0aDBs2DBqNBn369MGOHTus8BMlouZiEUVEDuell17C66+/jtTUVPTt2xelpaW4//77ER8fj+TkZIwePRpjx45FZmbmDc/zyiuv4LHHHsPhw4dx//33Y9KkSSgoKGgwtn379lAqlfj2229hMpkajNm3bx8A4LfffsOlS5fw3XffAQC++uorxMbG4rXXXkNqair++c9/YtGiRfj8888tjn/hhRcwd+5cJCcnIzIyEmPHjkV+fn5TfzxEZC2CiMhOffbZZ0Kr1Urfb9u2TQAQGzZsuOmxvXv3Fu+++670fXBwsPj3v/8tfQ9ALFy4UPq+tLRUABCbN2++7jnfe+894ebmJjw9PcVdd90lXn31VXHmzBlpf3p6ugAgkpOTLY7r0qWL+Prrry22/eMf/xCRkZEWx73++uvS/pqaGtGxY0fxxhtv3LStRGQb7IkiIoczaNAgi+9LS0vx/PPPo2fPntDpdPDw8EBqaupNe6L69u0r/dnd3R1eXl7Iycm5bvzMmTNhMBjw1VdfITIyEuvXr0fv3r0RFxd33WPKyspw5swZTJs2DR4eHtJryZIlOHPmjEVsZGSk9GdnZ2cMGjQIqampN2wDEdkOJ5YTkcNxd3e3+P75559HXFwcli9fjq5du8LV1RWPPPIIqqurb3geFxcXi+8VCgXMZvMNj/H09MTYsWMxduxYLFmyBNHR0ViyZAnuueeeBuNLS0sBAP/5z38QERFhsc/JyemG1yIiebEniogc3u7du/HEE0/goYceQnh4OPR6Pc6dO2fz6yoUCvTo0QNlZWUAAJVKBQAWc6b8/f0RGBiIs2fPomvXrhav+ono9fbu3Sv9uba2FklJSejZs6fN20FEDWNPFBE5vLCwMHz33XcYO3YsFAoFFi1adNMepaZKSUnB4sWL8fjjj6NXr15QqVTYsWMHPv30U8ybNw8A4OfnB1dXV2zZsgUdO3aERqOBVqvFK6+8gmeeeQZarRajR49GVVUVDhw4gMLCQsyZM0e6xsqVKxEWFoaePXvi3//+NwoLC/HXv/7Vqu0gosZjEUVEDu9f//oX/vrXv2LYsGHw9fXFvHnzYDQarXqNjh07IiQkBK+88oq0JEH998899xyAunlMK1aswKuvvorY2FiMGDEC27dvx//8z//Azc0Nb775Jl544QW4u7sjPDwcs2fPtrjG66+/jtdffx0pKSno2rUrfvzxR/j6+lq1HUTUeAohhJA7CSIiur5z584hNDQUycnJ6N+/v9zpENFlnBNFRERE1AwsooiIiIiagcN5RERERM3AnigiIiKiZmARRURERNQMLKKIiIiImoFFFBEREVEzsIgiIiIiagYWUURERETNwCKKiIiIqBlYRBERERE1w/8D5tOfkgv3fp4AAAAASUVORK5CYII=\n"},"metadata":{}}],"source":["plt.plot(learning_rate(tf.range(40000, dtype=tf.float32)))\n","plt.ylabel('Learning Rate')\n","plt.xlabel('Train Step')\n","plt.show()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Una1v0hDlIsT"},"outputs":[],"source":["transformer.compile(loss=masked_loss, optimizer=optimizer, metrics=[masked_accuracy])"]},{"cell_type":"markdown","source":["## 트랜스포머 학습"],"metadata":{"id":"vYauwuRTNC3Q"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"Jg35qKvVlctp"},"outputs":[],"source":["# GPU가 있어도 시간이 오래걸림\n","transformer.fit(train_batches, epochs=20, validation_data=val_batches)"]},{"cell_type":"markdown","source":["# 번역"],"metadata":{"id":"giNv3p7HNJwR"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"-NjbvpHUTEia"},"outputs":[],"source":["translator = Translator(tokenizers=tokenizers, transformer=transformer)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"o9CEm4cuTGtw"},"outputs":[],"source":["# Example 1:\n","sentence = 'este é um problema que temos que resolver.'\n","ground_truth = 'this is a problem we have to solve .'\n","\n","translated_text, translated_tokens, attention_weights = translator(\n","    tf.constant(sentence), max_length=MAX_TOKENS)\n","print_translation(sentence, translated_text, ground_truth)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"elmz_Ly7THuJ"},"outputs":[],"source":["# Example 2:\n","sentence = 'os meus vizinhos ouviram sobre esta ideia.'\n","ground_truth = 'and my neighboring homes heard about this idea .'\n","\n","translated_text, translated_tokens, attention_weights = translator(\n","    tf.constant(sentence), max_length=MAX_TOKENS)\n","print_translation(sentence, translated_text, ground_truth)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"bmmtPo3vTOwj"},"outputs":[],"source":["# Example 3:\n","sentence = 'vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.'\n","ground_truth = \"so i'll just share with you some stories very quickly of some magical things that have happened.\"\n","\n","translated_text, translated_tokens, attention_weights = translator(\n","    tf.constant(sentence), max_length=MAX_TOKENS)\n","print_translation(sentence, translated_text, ground_truth)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"V3m2wcNLTU8K"},"outputs":[],"source":["# Example 4:\n","sentence = 'este é o primeiro livro que eu fiz.'\n","ground_truth = \"this is the first book i've ever done.\"\n","\n","translated_text, translated_tokens, attention_weights = translator(\n","    tf.constant(sentence), max_length=MAX_TOKENS)\n","print_translation(sentence, translated_text, ground_truth)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"wm1_eRPvUCUm"},"outputs":[],"source":["translator = ExportTranslator(translator)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"GITRCiAYUE5w"},"outputs":[],"source":["translator('este é o primeiro livro que eu fiz.').numpy()"]},{"cell_type":"code","source":[],"metadata":{"id":"ec_Gi5YLotN_"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"last_runtime":{"build_target":"//learning/deepmind/public/tools/ml_python:ml_notebook","kind":"private"},"provenance":[{"file_id":"https://github.com/tensorflow/text/blob/master/docs/tutorials/transformer.ipynb","timestamp":1721855537419}],"toc_visible":true},"kernelspec":{"display_name":"Python 3","name":"python3"},"widgets":{"application/vnd.jupyter.widget-state+json":{"43276f2848d5450087b06a52e5fcb52e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5919574ff7524544a16b98ef60929ff5","IPY_MODEL_c3e74b1904134ddca349d03fe087b022","IPY_MODEL_4894798e1560439a9768d5dbe5af494d"],"layout":"IPY_MODEL_bfecdd93bcae458ea3515cdcd2285bfc"}},"5919574ff7524544a16b98ef60929ff5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b0522642574843ecbc8d13415bf7d876","placeholder":"​","style":"IPY_MODEL_f0a51618cfec4e1b94128f89be9bfc01","value":"Dl Completed...: 100%"}},"c3e74b1904134ddca349d03fe087b022":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_566c72dda8c24aa686e4deea5b684676","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8e496d04f0fc47eb94ecfd4eedb04249","value":1}},"4894798e1560439a9768d5dbe5af494d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5f8686fad8934440b97233b9db738a28","placeholder":"​","style":"IPY_MODEL_46761aa64c604e848373084e7a9cac9d","value":" 1/1 [00:13&lt;00:00,  4.75s/ url]"}},"bfecdd93bcae458ea3515cdcd2285bfc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b0522642574843ecbc8d13415bf7d876":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f0a51618cfec4e1b94128f89be9bfc01":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"566c72dda8c24aa686e4deea5b684676":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"8e496d04f0fc47eb94ecfd4eedb04249":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5f8686fad8934440b97233b9db738a28":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"46761aa64c604e848373084e7a9cac9d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"516237a3dfa444f1bcd313edc81fbc44":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_94cc956b897f444ea68820bbbe664dac","IPY_MODEL_32f0068103bc40f7b153655ae0f698ca","IPY_MODEL_117a45652f90414581aed1844128b68b"],"layout":"IPY_MODEL_8778da9ef3714955a65a9d7347ff7d6e"}},"94cc956b897f444ea68820bbbe664dac":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_fda55d7e78c94fe9877e19938e59fc42","placeholder":"​","style":"IPY_MODEL_d45345545cfb4fda8b5ea9222e6960d3","value":"Dl Size...: 100%"}},"32f0068103bc40f7b153655ae0f698ca":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_5a9d0b6ac2c84ffea2b503c718f8de06","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7538303a85194ee29d0cd5c8234da6db","value":1}},"117a45652f90414581aed1844128b68b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_522118041cfe4182a615d4cc9725d349","placeholder":"​","style":"IPY_MODEL_27c47044363d47fbb447da999a593926","value":" 124/124 [00:13&lt;00:00, 25.13 MiB/s]"}},"8778da9ef3714955a65a9d7347ff7d6e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fda55d7e78c94fe9877e19938e59fc42":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d45345545cfb4fda8b5ea9222e6960d3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5a9d0b6ac2c84ffea2b503c718f8de06":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"7538303a85194ee29d0cd5c8234da6db":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"522118041cfe4182a615d4cc9725d349":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"27c47044363d47fbb447da999a593926":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8d956d9b37644d12bc6b0abafecbd7d6":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_cecb63ff967d45669e94c206bb2d4076","IPY_MODEL_96deec1151fb4ffe81db1858f1058762","IPY_MODEL_2daa9bd0d6d3478b95c87e5afa6496db"],"layout":"IPY_MODEL_2e3bfbfa17ac4145a1e50903bccc5308"}},"cecb63ff967d45669e94c206bb2d4076":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c421a608e02b46beae73afd95a7430c8","placeholder":"​","style":"IPY_MODEL_234dc51befd74ba9a7af1b820d6074f5","value":"Extraction completed...: 100%"}},"96deec1151fb4ffe81db1858f1058762":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_49f96943370041a7be991adbecb64e97","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7694867b94394a67b8f0dbcfa08bcff0","value":1}},"2daa9bd0d6d3478b95c87e5afa6496db":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e3fae78dbe664d2f8c51a10f56248a7d","placeholder":"​","style":"IPY_MODEL_d88a837f69854dcf88d711865cf9cc97","value":" 112/112 [00:13&lt;00:00, 13.17s/ file]"}},"2e3bfbfa17ac4145a1e50903bccc5308":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c421a608e02b46beae73afd95a7430c8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"234dc51befd74ba9a7af1b820d6074f5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"49f96943370041a7be991adbecb64e97":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"7694867b94394a67b8f0dbcfa08bcff0":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e3fae78dbe664d2f8c51a10f56248a7d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d88a837f69854dcf88d711865cf9cc97":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"10a00c5e760e4b23a81f19b78bcb8163":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_727fbc26e9dd4a81b39ed4d796d4bd95","IPY_MODEL_320ebfe94e264f5194881de168f8c27b","IPY_MODEL_3215b96a369a418ba0762d3bec83c88d"],"layout":"IPY_MODEL_8e656799220b463aa1e4e1cef8bec17e"}},"727fbc26e9dd4a81b39ed4d796d4bd95":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d01cb942e8664f139524fa42737d7858","placeholder":"​","style":"IPY_MODEL_f45904c00d4046adab624800596aa153","value":"Generating splits...: 100%"}},"320ebfe94e264f5194881de168f8c27b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_ae2ae93367144d02b49dd8d9e1f0118b","max":3,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d0b0096959de4dfeb695200974b0e622","value":3}},"3215b96a369a418ba0762d3bec83c88d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3121ffd5d95b4866b76436a5fe47a270","placeholder":"​","style":"IPY_MODEL_211196069e224e3eaddcf8cfb7f06601","value":" 3/3 [00:14&lt;00:00,  3.48s/ splits]"}},"8e656799220b463aa1e4e1cef8bec17e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"d01cb942e8664f139524fa42737d7858":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f45904c00d4046adab624800596aa153":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ae2ae93367144d02b49dd8d9e1f0118b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d0b0096959de4dfeb695200974b0e622":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"3121ffd5d95b4866b76436a5fe47a270":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"211196069e224e3eaddcf8cfb7f06601":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"69758a2e414944c3b00765b7f433a95d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ae620515eaef4ee1962b7f73e4bfcb76","IPY_MODEL_433a1d3ca73a42ccbbb8670df14cc70c","IPY_MODEL_92162094ca7f408692e097f40c713b77"],"layout":"IPY_MODEL_0f428af76f234a99871277ea2964c5a5"}},"ae620515eaef4ee1962b7f73e4bfcb76":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0f17fc9272134131a4747a964892fc89","placeholder":"​","style":"IPY_MODEL_5d2bf0f5b6e64e5bbafb9a42e777bbf2","value":"Generating train examples...:  97%"}},"433a1d3ca73a42ccbbb8670df14cc70c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_c39e2afb29a84d8bb197c1aea0b499ef","max":51785,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6bad5f8039094968b79a26cb5ae23288","value":51785}},"92162094ca7f408692e097f40c713b77":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7fee9841b9c8417bbb79fa7236809748","placeholder":"​","style":"IPY_MODEL_35dd13bfd6e34022abaee971ceae47ac","value":" 50291/51785 [00:12&lt;00:00, 5119.80 examples/s]"}},"0f428af76f234a99871277ea2964c5a5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"0f17fc9272134131a4747a964892fc89":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5d2bf0f5b6e64e5bbafb9a42e777bbf2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c39e2afb29a84d8bb197c1aea0b499ef":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6bad5f8039094968b79a26cb5ae23288":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7fee9841b9c8417bbb79fa7236809748":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"35dd13bfd6e34022abaee971ceae47ac":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9031896bd7d443e7b98d39a3351d2566":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_562ab14e85994040b37ce863c01bfb7d","IPY_MODEL_d2095fdb584a4ad7b813b69ae499e192","IPY_MODEL_c7287e704cad437c9b1f52902c6f6147"],"layout":"IPY_MODEL_a51ad239145641ae9912d5784b70d396"}},"562ab14e85994040b37ce863c01bfb7d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_211cfb647fc34fb990f9a10001367f4c","placeholder":"​","style":"IPY_MODEL_8f5e30185e7a43689cca17d26136d325","value":"Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-train.tfrecord*...:   0%"}},"d2095fdb584a4ad7b813b69ae499e192":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_48eab2762fdb4c69b5e49a9546103a1f","max":51785,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d35ca563c8b64c368b64317ca9830421","value":51785}},"c7287e704cad437c9b1f52902c6f6147":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6937dd0896a44933ba1a63f6988af596","placeholder":"​","style":"IPY_MODEL_ebb1468294a14e4384d7921a9ec95429","value":" 0/51785 [00:00&lt;?, ? examples/s]"}},"a51ad239145641ae9912d5784b70d396":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"211cfb647fc34fb990f9a10001367f4c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8f5e30185e7a43689cca17d26136d325":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"48eab2762fdb4c69b5e49a9546103a1f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d35ca563c8b64c368b64317ca9830421":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6937dd0896a44933ba1a63f6988af596":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ebb1468294a14e4384d7921a9ec95429":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"55cc534b52f9453fa6880f9259f97179":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fc1bc2e98e5e42358fef5b5e22318901","IPY_MODEL_0cade649c70e4985915c1cc32db481bb","IPY_MODEL_62e34718a06649ffaad33074cc4d5bd3"],"layout":"IPY_MODEL_b343cb3a319a4aed82ad123a151b29fa"}},"fc1bc2e98e5e42358fef5b5e22318901":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_998c375c01e54029b899090d6847cb9b","placeholder":"​","style":"IPY_MODEL_6014658bba0748869115a16b84760f6a","value":"Generating validation examples...:   0%"}},"0cade649c70e4985915c1cc32db481bb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_a3918bbef7704ccdb65c546cd28b277c","max":1193,"min":0,"orientation":"horizontal","style":"IPY_MODEL_303889f13e7b40aaa97fc36166b794da","value":1193}},"62e34718a06649ffaad33074cc4d5bd3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b1aa8e164d2c46f19b1677b6df85b820","placeholder":"​","style":"IPY_MODEL_9bd4959bf13748bd818acfb947b2cfce","value":" 0/1193 [00:00&lt;?, ? examples/s]"}},"b343cb3a319a4aed82ad123a151b29fa":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"998c375c01e54029b899090d6847cb9b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6014658bba0748869115a16b84760f6a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a3918bbef7704ccdb65c546cd28b277c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"303889f13e7b40aaa97fc36166b794da":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b1aa8e164d2c46f19b1677b6df85b820":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9bd4959bf13748bd818acfb947b2cfce":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1fdd342c87514b52ae1715e7e60c60e1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5250c7c093f44e41a4db605af76e6115","IPY_MODEL_e27420d9d3e3489294ea575f837f1134","IPY_MODEL_8f823403ebc0424e9c7f234f4331dcf1"],"layout":"IPY_MODEL_4b8a60042ea845879b30b0a17e404620"}},"5250c7c093f44e41a4db605af76e6115":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_fde787f1305048c696a9c2fd39a9a6b1","placeholder":"​","style":"IPY_MODEL_84eb98c3b4a6443bb929faac041fa210","value":"Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-validation.tfrecord*...:   0%"}},"e27420d9d3e3489294ea575f837f1134":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_d4062553fc6547488563451eddeb9cb7","max":1193,"min":0,"orientation":"horizontal","style":"IPY_MODEL_95632fd4607e4c14878abc11c432144e","value":1193}},"8f823403ebc0424e9c7f234f4331dcf1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_686682c4d9a24f94ba19fa1838177251","placeholder":"​","style":"IPY_MODEL_803673935a0c4f5abb5246c0c5cd3904","value":" 0/1193 [00:00&lt;?, ? examples/s]"}},"4b8a60042ea845879b30b0a17e404620":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"fde787f1305048c696a9c2fd39a9a6b1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"84eb98c3b4a6443bb929faac041fa210":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d4062553fc6547488563451eddeb9cb7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"95632fd4607e4c14878abc11c432144e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"686682c4d9a24f94ba19fa1838177251":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"803673935a0c4f5abb5246c0c5cd3904":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"60a010ec3e7a4541972df5419ed1961e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e2f0e147902340cda73996119ef9de8f","IPY_MODEL_b02090a0272b49d19502aea2ff72deea","IPY_MODEL_24a66ce200404ac6ac5353c1c503b299"],"layout":"IPY_MODEL_38b8b6856b854a9e8adfa687dd86941e"}},"e2f0e147902340cda73996119ef9de8f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5498751a235b441b989b5a6c4f4127e9","placeholder":"​","style":"IPY_MODEL_2f11e674139442ea884d26aebf43af4a","value":"Generating test examples...:   0%"}},"b02090a0272b49d19502aea2ff72deea":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_80bc51204710477394cd0049ef2dc2b3","max":1803,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5da90bd31cec4c49bade659a725d1480","value":1803}},"24a66ce200404ac6ac5353c1c503b299":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_111648081f694975afaef411933e99b0","placeholder":"​","style":"IPY_MODEL_d3f4d62262244237bf052cdb370f0d67","value":" 0/1803 [00:00&lt;?, ? examples/s]"}},"38b8b6856b854a9e8adfa687dd86941e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"5498751a235b441b989b5a6c4f4127e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2f11e674139442ea884d26aebf43af4a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"80bc51204710477394cd0049ef2dc2b3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5da90bd31cec4c49bade659a725d1480":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"111648081f694975afaef411933e99b0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d3f4d62262244237bf052cdb370f0d67":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f9ef65e4de9d4c8987a6fe9f4f93188a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_41094d8baa5242ebb4f89955f9530616","IPY_MODEL_7be7d795450f4e17a2d5b6838df17f34","IPY_MODEL_9463853cb5d7485fa939fc79b0c385fb"],"layout":"IPY_MODEL_b9a3b93a76d646b7b2cb05af5160da94"}},"41094d8baa5242ebb4f89955f9530616":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0a7fc203034343faa4486a7a0a682746","placeholder":"​","style":"IPY_MODEL_83b73ede65f440aaa35a526498def1ce","value":"Shuffling /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/incomplete.I8735R_1.0.0/ted_hrlr_translate-test.tfrecord*...:   0%"}},"7be7d795450f4e17a2d5b6838df17f34":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_e82c7d973eb443e3bbea25be19e873db","max":1803,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c0731bc2210d413f848a7dadf8373255","value":1803}},"9463853cb5d7485fa939fc79b0c385fb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_25402a23331042b3acd1a9861509e0d1","placeholder":"​","style":"IPY_MODEL_ca10d72db12f4e9b9ef42efedae2a94c","value":" 0/1803 [00:00&lt;?, ? examples/s]"}},"b9a3b93a76d646b7b2cb05af5160da94":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"0a7fc203034343faa4486a7a0a682746":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"83b73ede65f440aaa35a526498def1ce":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e82c7d973eb443e3bbea25be19e873db":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c0731bc2210d413f848a7dadf8373255":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"25402a23331042b3acd1a9861509e0d1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ca10d72db12f4e9b9ef42efedae2a94c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}