layers.0.mlp/cfg.json

{"activation": "topk", "expansion_factor": 32, "normalize_decoder": true, "num_latents": 131072, "k": 32, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 2048}