PackedLMData.Config

Component: PackedLMData

class PackedLMData.Config

Bases: Data.Config

All Attributes (including base classes)

source: DataSource.Config = TSVDataSource.Config()

batcher: Batcher.Config = PoolingBatcher.Config()

sort_key: Optional[str] = None

in_memory: Optional[bool] = True

max_seq_len: int = 128

Default JSON

{
    "source": {
        "TSVDataSource": {
            "column_mapping": {},
            "train_filename": null,
            "test_filename": null,
            "eval_filename": null,
            "field_names": null,
            "delimiter": "\t",
            "quoted": false,
            "drop_incomplete_rows": false
        }
    },
    "batcher": {
        "PoolingBatcher": {
            "train_batch_size": 16,
            "eval_batch_size": 16,
            "test_batch_size": 16,
            "pool_num_batches": 1000,
            "num_shuffled_pools": 1
        }
    },
    "sort_key": null,
    "in_memory": true,
    "max_seq_len": 128
}