ByteModelInput

class pytext.models.word_model.ByteModelInput

Bases: ModelInput

All Attributes (including base classes)

token_bytes: ByteTokenTensorizer.Config = ByteTokenTensorizer.Config()

labels: SlotLabelTensorizer.Config = SlotLabelTensorizer.Config()

Default JSON

{
    "token_bytes": {
        "is_input": true,
        "column": "text",
        "tokenizer": {
            "Tokenizer": {
                "split_regex": "\\s+",
                "lowercase": true,
                "use_byte_offsets": false
            }
        },
        "max_seq_len": null,
        "max_byte_len": 15,
        "offset_for_non_padding": 0,
        "add_bos_token": false,
        "add_eos_token": false,
        "use_eos_token_for_bos": false
    },
    "labels": {
        "is_input": false,
        "slot_column": "slots",
        "text_column": "text",
        "tokenizer": {
            "Tokenizer": {
                "split_regex": "\\s+",
                "lowercase": true,
                "use_byte_offsets": false
            }
        },
        "allow_unknown": false
    }
}