Hugging Face's logo

inclusionAI
/
Ling-plus-base

Text Generation
Transformers
Safetensors
bailing_moe
conversational
custom_code
Model card Files Community
2
lushuai commited on
Commit
fbbab26
·
1 Parent(s): c6b14bc

init repo

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +44 -0
  2. configuration_bailing_moe.py +78 -0
  3. model-00001-of-00136.safetensors +3 -0
  4. model-00002-of-00136.safetensors +3 -0
  5. model-00003-of-00136.safetensors +3 -0
  6. model-00004-of-00136.safetensors +3 -0
  7. model-00005-of-00136.safetensors +3 -0
  8. model-00006-of-00136.safetensors +3 -0
  9. model-00007-of-00136.safetensors +3 -0
  10. model-00008-of-00136.safetensors +3 -0
  11. model-00009-of-00136.safetensors +3 -0
  12. model-00010-of-00136.safetensors +3 -0
  13. model-00011-of-00136.safetensors +3 -0
  14. model-00012-of-00136.safetensors +3 -0
  15. model-00013-of-00136.safetensors +3 -0
  16. model-00014-of-00136.safetensors +3 -0
  17. model-00015-of-00136.safetensors +3 -0
  18. model-00016-of-00136.safetensors +3 -0
  19. model-00017-of-00136.safetensors +3 -0
  20. model-00018-of-00136.safetensors +3 -0
  21. model-00019-of-00136.safetensors +3 -0
  22. model-00020-of-00136.safetensors +3 -0
  23. model-00021-of-00136.safetensors +3 -0
  24. model-00022-of-00136.safetensors +3 -0
  25. model-00023-of-00136.safetensors +3 -0
  26. model-00024-of-00136.safetensors +3 -0
  27. model-00025-of-00136.safetensors +3 -0
  28. model-00026-of-00136.safetensors +3 -0
  29. model-00027-of-00136.safetensors +3 -0
  30. model-00028-of-00136.safetensors +3 -0
  31. model-00029-of-00136.safetensors +3 -0
  32. model-00030-of-00136.safetensors +3 -0
  33. model-00031-of-00136.safetensors +3 -0
  34. model-00032-of-00136.safetensors +3 -0
  35. model-00033-of-00136.safetensors +3 -0
  36. model-00034-of-00136.safetensors +3 -0
  37. model-00035-of-00136.safetensors +3 -0
  38. model-00036-of-00136.safetensors +3 -0
  39. model-00037-of-00136.safetensors +3 -0
  40. model-00038-of-00136.safetensors +3 -0
  41. model-00039-of-00136.safetensors +3 -0
  42. model-00040-of-00136.safetensors +3 -0
  43. model-00041-of-00136.safetensors +3 -0
  44. model-00042-of-00136.safetensors +3 -0
  45. model-00043-of-00136.safetensors +3 -0
  46. model-00044-of-00136.safetensors +3 -0
  47. model-00045-of-00136.safetensors +3 -0
  48. model-00046-of-00136.safetensors +3 -0
  49. model-00047-of-00136.safetensors +3 -0
  50. model-00048-of-00136.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BailingMoeForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_bailing_moe.BailingMoeConfig",
8
+ "AutoModel": "modeling_bailing_moe.BailingMoeModel",
9
+ "AutoModelForCausalLM": "modeling_bailing_moe.BailingMoeForCausalLM"
10
+ },
11
+ "eos_token_id": 126081,
12
+ "pad_token_id": 126081,
13
+ "first_k_dense_replace": 0,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 5376,
16
+ "initializer_range": 0.006,
17
+ "intermediate_size": 12288,
18
+ "max_position_embeddings": 16384,
19
+ "model_type": "bailing_moe",
20
+ "moe_intermediate_size": 3072,
21
+ "num_experts": 64,
22
+ "num_shared_experts": 1,
23
+ "norm_topk_prob": true,
24
+ "num_attention_heads": 42,
25
+ "num_experts_per_tok": 4,
26
+ "num_hidden_layers": 88,
27
+ "num_key_value_heads": 6,
28
+ "pretraining_tp": 1,
29
+ "rms_norm_eps": 1e-06,
30
+ "rope_scaling": null,
31
+ "rope_theta": 600000,
32
+ "tie_word_embeddings": false,
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.36.0",
35
+ "use_cache": true,
36
+ "use_bias": false,
37
+ "use_qkv_bias": false,
38
+ "vocab_size": 126464,
39
+ "embedding_dropout": 0.0,
40
+ "norm_head": true,
41
+ "norm_softmax": false,
42
+ "output_dropout": 0.0,
43
+ "output_router_logits": false
44
+ }
configuration_bailing_moe.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ Bailing MoE model configuration """
2
+
3
+ from transformers.configuration_utils import PretrainedConfig
4
+
5
+
6
+ class BailingMoeConfig(PretrainedConfig):
7
+ model_type = "bailing_moe"
8
+
9
+ def __init__(
10
+ self,
11
+ vocab_size=30592,
12
+ hidden_size=1024,
13
+ intermediate_size=None,
14
+ num_hidden_layers=24,
15
+ num_attention_heads=16,
16
+ num_key_value_heads=0,
17
+ hidden_act="silu",
18
+ use_qkv_bias=False, # bailing only
19
+ use_bias=True, # bailing only
20
+ rms_norm_eps=1e-05,
21
+ norm_head=False, # bailing only
22
+ tie_word_embeddings=False, # PretrainedConfig key, here change default value.
23
+ embedding_dropout=0.1,
24
+ attention_dropout=0.1,
25
+ output_dropout=0.1,
26
+ initializer_range=0.02,
27
+ max_position_embeddings=16384,
28
+ rope_theta=10000.0,
29
+ use_cache=True,
30
+ use_sliding_window=False,
31
+ sliding_window=4096,
32
+ max_window_layers=28,
33
+ rope_scaling=None,
34
+ pad_token_id=126081,
35
+ num_experts=16,
36
+ num_shared_experts=0,
37
+ num_experts_per_tok=2,
38
+ norm_topk_prob=True,
39
+ moe_intermediate_size=None,
40
+ first_k_dense_replace=0,
41
+ head_dim=None,
42
+ output_router_logits=False,
43
+ **kwargs,
44
+ ):
45
+ self.num_hidden_layers = num_hidden_layers
46
+ self.vocab_size = vocab_size
47
+ self.hidden_size = hidden_size
48
+ self.intermediate_size = intermediate_size
49
+ self.num_attention_heads = num_attention_heads
50
+ self.num_key_value_heads = num_key_value_heads
51
+ self.hidden_act = hidden_act
52
+ self.use_qkv_bias = use_qkv_bias
53
+ self.use_bias = use_bias
54
+ self.norm_head = norm_head
55
+ self.rms_norm_eps = rms_norm_eps
56
+ self.embedding_dropout = embedding_dropout
57
+ self.attention_dropout = attention_dropout
58
+ self.output_dropout = output_dropout
59
+ self.initializer_range = initializer_range
60
+ self.max_position_embeddings = max_position_embeddings
61
+ self.rope_theta = rope_theta
62
+ self.use_cache = use_cache
63
+ self.use_sliding_window = use_sliding_window
64
+ self.sliding_window = sliding_window
65
+ self.max_window_layers = max_window_layers
66
+ self.head_dim = head_dim
67
+ self.rope_scaling = rope_scaling
68
+
69
+ # MoE configs
70
+ self.num_experts = num_experts
71
+ self.num_shared_experts = num_shared_experts
72
+ self.num_experts_per_tok = num_experts_per_tok
73
+ self.norm_topk_prob = norm_topk_prob
74
+ self.moe_intermediate_size = moe_intermediate_size
75
+ self.first_k_dense_replace = first_k_dense_replace
76
+ self.output_router_logits = output_router_logits
77
+
78
+ super().__init__(pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)
model-00001-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6a15d616c0c7a65e7752aaf3f3bff189a3f9d4eb28be265fb32a6accc9f9c71
3
+ size 3622335840
model-00002-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a86675330958c8018b1be09e6bfd66569127b3464c0c7c0111419045dcbc528c
3
+ size 4376532008
model-00003-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18eca4e9559df276a095077ed50320d440836cde2048f117acdf9bab667f42e5
3
+ size 4376532008
model-00004-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b3da6f76d2ab65eb05dcfaaac06e402a25fec1132d60f489bdcbc9a1efa2afb
3
+ size 4376532008
model-00005-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5036085fb6129938182e85b7b7d6c3628fe10c6e32b97bfc4249ce5fc0d3d32
3
+ size 4376532008
model-00006-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28284dea3d9aee202feed3a3392f7699de60db23e9b49b47349baa652591d6e3
3
+ size 4376532080
model-00007-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33a70e24b1e9fa3ddf29893b5eecd04efa339dfb0752724b8cd468fe74f1f7c7
3
+ size 4376532136
model-00008-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4d620332481c19db21da1c5fd47ceaaae6e374b3a8759494c4fb4b7609f7a63
3
+ size 4376532136
model-00009-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d46f99805058104127044e30cf8b4e5793ec9e22cf9acbadafde70831d9dd98d
3
+ size 4376532136
model-00010-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31101303d87d7aadc15f132ba057de97ad765711b01017f229db1378c2ce70c4
3
+ size 4376532136
model-00011-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02492b918ae3538c248991154f3347e8763f3531681ccf4da9f45d5e82c91e12
3
+ size 4376532136
model-00012-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73fb58757c12b762a7c7b80680305bd7849228496a00f91d927bf018d99e34f5
3
+ size 4376532136
model-00013-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:443450811bea17b1f6697bfd2bf37f56d069c88f0d2c33353da8c63b52aa276d
3
+ size 4376532136
model-00014-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0750dd07a9e10758018f4e895599ee9e97f71fbb72d222564efe0756a89cda6c
3
+ size 4376532136
model-00015-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19530973fa9ccff9d8e54444f6997b09986223bb18c4e29f253f725a504ddd9f
3
+ size 4376532136
model-00016-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4364ed56e0a4532551e0bf4b304d5e0f4640c701da4eb2440e81ce5d579ab88c
3
+ size 4376532136
model-00017-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9140e9e3a001e50a3346b287fa3c5857e3734dfbaad0bb1a7bfc7118d17b214
3
+ size 4376532136
model-00018-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd28252d2d6b3e1bb5c208b666765f90e5dee44bbdf9ebd111da788cafaa9c17
3
+ size 4376532136
model-00019-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:230ed5bf5d5b147d50852ce74b16a14580e71a9eb81153dae72753029d9b900b
3
+ size 4376532136
model-00020-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9b2f3743abd955b20bf50bbd57837b8a88363abc7def2e589908ebc14a09db4
3
+ size 4376532136
model-00021-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8137c48fc14479ded5710e35bb0cd360bde442b03865c61a6487436f5ef4c0a
3
+ size 4376532136
model-00022-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2c1fb89204f6d0f716900f046379d4c7483fd0f98e848b8d0471872e6dbbd7c
3
+ size 4376532136
model-00023-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bf178a4ae6720fff982f7ac92f757532364f4b43492fd08e64832f9605b179e
3
+ size 4376532136
model-00024-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ab70fea330dc47a2fea057dab3aebed5127b067f10f3e51bbf8f2f8f91e1a03
3
+ size 4376532136
model-00025-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:066fed1d407581399754e2d1a42abdee73a807c760af8f4b5c0ad06d4b52da15
3
+ size 4376532136
model-00026-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ea778b8318f4923d27c110f7dfaffe3975d5d054bdea1561ec025d3f37232a9
3
+ size 4376532136
model-00027-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6eac84b1e66897282b8f6b2a611655669bf5ba260f665ec5a5ec4ce2ee3e9827
3
+ size 4376532136
model-00028-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28c7f50b95d8d0342b91d85736d755e2e0ecdf6078e21b36e5686482cfb54f7e
3
+ size 4376532136
model-00029-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d77df85a104ace107c8c7af5e10481751efbc272e9ddeb7d2504a47c5ca702a4
3
+ size 4376532136
model-00030-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3435b00b747f3f15d703b3fdc1f1fc1076116259ef25d73f3f89965eb03d6c32
3
+ size 4376532136
model-00031-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d51be03cd112238f6a175cb4884a991c9d1380aaaf429a64a35389ff5cf83290
3
+ size 4376532136
model-00032-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f134bb0ba8c67a8cbd6f39bba910abbf98a5794d08afe6d5c2992245727c5fce
3
+ size 4376532136
model-00033-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eea738b166db0baf4cd832ebfddbe3df0cb9ca502f0dbeb9ada7eaf5acdeb42c
3
+ size 4376532136
model-00034-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d116c5def59f6ef9b70f822a48ad2448e74558e84059117ec4c522c1465cb897
3
+ size 4376532136
model-00035-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70a09ea68425b478e647aa569c5dfeb6d03326ae596289397bf2bee15401f651
3
+ size 4376532136
model-00036-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b70a8b0a14b29d003bf946f97738fe1b34c527c791005789cddf9b6d8c32b52
3
+ size 4376532136
model-00037-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a494a26d9b8f29e93d1b67d5df7cbad43107234c27067c93ae44c671147fae10
3
+ size 4376532136
model-00038-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32d873bf8686554e7b30f62e70b2f5198185e8573c3102c340b3c344d1f0e205
3
+ size 4376532136
model-00039-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2608d98681a4c70c978677b8f2fba33d5d80e6892c6c10f8ccca1edbf3cd9b05
3
+ size 4376532136
model-00040-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc82db6958c9ce83c2b8203fb8814dd5d3a4c4a2166ea303311bd0ac2a6cdd99
3
+ size 4376532136
model-00041-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ad833ce919c65186b3086c91b706ab9b5e3e5fd1fe47acdabed5a1a203a489a
3
+ size 4376532136
model-00042-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:726d9a6984aad311668e5d5f3667ff9228007e486ee9fecc770e84feafbaae2e
3
+ size 4376532136
model-00043-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7873a5c81148565a7b0607d605f64b49a8c7c383590365f5fa00e604f99f8de1
3
+ size 4376532136
model-00044-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f1f6e632629600b0d6a49dc1b2ad0bb7c01ce449b6a4a32adbb725096d64638
3
+ size 4376532136
model-00045-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adb4436b4f581a6dfad93c26d636bf1699d22ced29e2cd03c757145101df2723
3
+ size 4946277840
model-00046-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65f4583a6ad45d8bd8b65841199da6eb89591f2435f43a5de810bbfc917109a6
3
+ size 4368249064
model-00047-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b29344bb2f8f342f104668a4066229a15f4ae13427b15bcac48bf3bdacddeefb
3
+ size 4227874384
model-00048-of-00136.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d7e9a9279fa56f3eda2d9eb07de37d471909866edbf906381a8c34e048f809c
3
+ size 4227874384