MaskedLanguageModel.Config

Component: MaskedLanguageModel

class MaskedLanguageModel.Config

Bases: BaseModel.Config

All Attributes (including base classes)

inputs: InputConfig = InputConfig()

encoder: TransformerSentenceEncoderBase.Config = TransformerSentenceEncoder.Config()

decoder: MLPDecoder.Config = MLPDecoder.Config()

output_layer: LMOutputLayer.Config = LMOutputLayer.Config()

mask_prob: float = 0.15

mask_bos: bool = False

masking_strategy: MaskingStrategy = <MaskingStrategy.RANDOM: 'random'>

tie_weights: bool = True

Default JSON

{
    "inputs": {
        "tokens": {
            "BERTTensorizerBase": {
                "is_input": true,
                "columns": [
                    "text"
                ],
                "tokenizer": {
                    "Tokenizer": {
                        "split_regex": "\\s+",
                        "lowercase": true,
                        "use_byte_offsets": false
                    }
                },
                "base_tokenizer": null,
                "vocab_file": "",
                "max_seq_len": 128
            }
        }
    },
    "encoder": {
        "TransformerSentenceEncoder": {
            "load_path": null,
            "save_path": null,
            "freeze": false,
            "shared_module_key": null,
            "output_dropout": 0.4,
            "embedding_dim": 768,
            "pooling": "cls_token",
            "export": false,
            "projection_dim": 0,
            "normalize_output_rep": false,
            "dropout": 0.1,
            "attention_dropout": 0.1,
            "activation_dropout": 0.1,
            "ffn_embedding_dim": 3072,
            "num_encoder_layers": 6,
            "num_attention_heads": 8,
            "num_segments": 2,
            "use_position_embeddings": true,
            "offset_positions_by_padding": true,
            "apply_bert_init": true,
            "encoder_normalize_before": true,
            "activation_fn": "relu",
            "max_seq_len": 128,
            "multilingual": false,
            "freeze_embeddings": false,
            "n_trans_layers_to_freeze": 0,
            "use_torchscript": false,
            "use_bias_finetuning": false
        }
    },
    "decoder": {
        "load_path": null,
        "save_path": null,
        "freeze": false,
        "shared_module_key": null,
        "hidden_dims": [],
        "out_dim": null,
        "layer_norm": false,
        "dropout": 0.0,
        "bias": true,
        "activation": "relu",
        "temperature": 1.0,
        "spectral_normalization": false
    },
    "output_layer": {
        "load_path": null,
        "save_path": null,
        "freeze": false,
        "shared_module_key": null,
        "loss": {}
    },
    "mask_prob": 0.15,
    "mask_bos": false,
    "masking_strategy": "random",
    "tie_weights": true
}