Hugging Face's logo

Spaces:
janmayjay
/
demo
Sleeping

App Files Community
janmayjay commited on
Commit
7b53f4b
·
1 Parent(s): 9adc748

Your commit message

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. app.py +0 -1
  2. requirements.txt +9 -0
  3. smollm2-finetuned-lora1/README.md +207 -0
  4. smollm2-finetuned-lora1/adapter_config.json +36 -0
  5. smollm2-finetuned-lora1/adapter_model.safetensors +3 -0
  6. smollm2-finetuned-lora1/merges.txt +0 -0
  7. smollm2-finetuned-lora1/special_tokens_map.json +43 -0
  8. smollm2-finetuned-lora1/tokenizer.json +0 -0
  9. smollm2-finetuned-lora1/tokenizer_config.json +169 -0
  10. smollm2-finetuned-lora1/vocab.json +0 -0
  11. smollm2-finetuned1/checkpoint-192/README.md +207 -0
  12. smollm2-finetuned1/checkpoint-192/adapter_config.json +36 -0
  13. smollm2-finetuned1/checkpoint-192/adapter_model.safetensors +3 -0
  14. smollm2-finetuned1/checkpoint-192/merges.txt +0 -0
  15. smollm2-finetuned1/checkpoint-192/optimizer.pt +3 -0
  16. smollm2-finetuned1/checkpoint-192/rng_state.pth +3 -0
  17. smollm2-finetuned1/checkpoint-192/scaler.pt +3 -0
  18. smollm2-finetuned1/checkpoint-192/scheduler.pt +3 -0
  19. smollm2-finetuned1/checkpoint-192/special_tokens_map.json +43 -0
  20. smollm2-finetuned1/checkpoint-192/tokenizer.json +0 -0
  21. smollm2-finetuned1/checkpoint-192/tokenizer_config.json +169 -0
  22. smollm2-finetuned1/checkpoint-192/trainer_state.json +167 -0
  23. smollm2-finetuned1/checkpoint-192/training_args.bin +3 -0
  24. smollm2-finetuned1/checkpoint-192/vocab.json +0 -0
  25. smollm2-finetuned1/checkpoint-384/README.md +207 -0
  26. smollm2-finetuned1/checkpoint-384/adapter_config.json +36 -0
  27. smollm2-finetuned1/checkpoint-384/adapter_model.safetensors +3 -0
  28. smollm2-finetuned1/checkpoint-384/merges.txt +0 -0
  29. smollm2-finetuned1/checkpoint-384/optimizer.pt +3 -0
  30. smollm2-finetuned1/checkpoint-384/rng_state.pth +3 -0
  31. smollm2-finetuned1/checkpoint-384/scaler.pt +3 -0
  32. smollm2-finetuned1/checkpoint-384/scheduler.pt +3 -0
  33. smollm2-finetuned1/checkpoint-384/special_tokens_map.json +43 -0
  34. smollm2-finetuned1/checkpoint-384/tokenizer.json +0 -0
  35. smollm2-finetuned1/checkpoint-384/tokenizer_config.json +169 -0
  36. smollm2-finetuned1/checkpoint-384/trainer_state.json +300 -0
  37. smollm2-finetuned1/checkpoint-384/training_args.bin +3 -0
  38. smollm2-finetuned1/checkpoint-384/vocab.json +0 -0
  39. smollm2-finetuned1/checkpoint-576/README.md +207 -0
  40. smollm2-finetuned1/checkpoint-576/adapter_config.json +36 -0
  41. smollm2-finetuned1/checkpoint-576/adapter_model.safetensors +3 -0
  42. smollm2-finetuned1/checkpoint-576/merges.txt +0 -0
  43. smollm2-finetuned1/checkpoint-576/optimizer.pt +3 -0
  44. smollm2-finetuned1/checkpoint-576/rng_state.pth +3 -0
  45. smollm2-finetuned1/checkpoint-576/scaler.pt +3 -0
  46. smollm2-finetuned1/checkpoint-576/scheduler.pt +3 -0
  47. smollm2-finetuned1/checkpoint-576/special_tokens_map.json +43 -0
  48. smollm2-finetuned1/checkpoint-576/tokenizer.json +0 -0
  49. smollm2-finetuned1/checkpoint-576/tokenizer_config.json +169 -0
  50. smollm2-finetuned1/checkpoint-576/trainer_state.json +433 -0
app.py CHANGED
@@ -1,6 +1,5 @@
1
  # Install required packages (for Colab/CLI)
2
  # !pip install -U bitsandbytes langchain pypdf peft transformers accelerate datasets langchain-community faiss-cpu gradio
3
- !pip install gradio langchain transformers peft accelerate bitsandbytes faiss-cpu pypdf langchain-community
4
 
5
  import gradio as gr
6
  import torch
 
1
  # Install required packages (for Colab/CLI)
2
  # !pip install -U bitsandbytes langchain pypdf peft transformers accelerate datasets langchain-community faiss-cpu gradio
 
3
 
4
  import gradio as gr
5
  import torch
requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ gradio
2
+ langchain
3
+ transformers
4
+ peft
5
+ accelerate
6
+ bitsandbytes
7
+ faiss-cpu
8
+ pypdf
9
+ langchain-community
smollm2-finetuned-lora1/README.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: HuggingFaceTB/SmolLM2-360M
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:HuggingFaceTB/SmolLM2-360M
7
+ - lora
8
+ - transformers
9
+ ---
10
+
11
+ # Model Card for Model ID
12
+
13
+ <!-- Provide a quick summary of what the model is/does. -->
14
+
15
+
16
+
17
+ ## Model Details
18
+
19
+ ### Model Description
20
+
21
+ <!-- Provide a longer summary of what this model is. -->
22
+
23
+
24
+
25
+ - **Developed by:** [More Information Needed]
26
+ - **Funded by [optional]:** [More Information Needed]
27
+ - **Shared by [optional]:** [More Information Needed]
28
+ - **Model type:** [More Information Needed]
29
+ - **Language(s) (NLP):** [More Information Needed]
30
+ - **License:** [More Information Needed]
31
+ - **Finetuned from model [optional]:** [More Information Needed]
32
+
33
+ ### Model Sources [optional]
34
+
35
+ <!-- Provide the basic links for the model. -->
36
+
37
+ - **Repository:** [More Information Needed]
38
+ - **Paper [optional]:** [More Information Needed]
39
+ - **Demo [optional]:** [More Information Needed]
40
+
41
+ ## Uses
42
+
43
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
44
+
45
+ ### Direct Use
46
+
47
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
48
+
49
+ [More Information Needed]
50
+
51
+ ### Downstream Use [optional]
52
+
53
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
54
+
55
+ [More Information Needed]
56
+
57
+ ### Out-of-Scope Use
58
+
59
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
60
+
61
+ [More Information Needed]
62
+
63
+ ## Bias, Risks, and Limitations
64
+
65
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
66
+
67
+ [More Information Needed]
68
+
69
+ ### Recommendations
70
+
71
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
72
+
73
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
74
+
75
+ ## How to Get Started with the Model
76
+
77
+ Use the code below to get started with the model.
78
+
79
+ [More Information Needed]
80
+
81
+ ## Training Details
82
+
83
+ ### Training Data
84
+
85
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
86
+
87
+ [More Information Needed]
88
+
89
+ ### Training Procedure
90
+
91
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
92
+
93
+ #### Preprocessing [optional]
94
+
95
+ [More Information Needed]
96
+
97
+
98
+ #### Training Hyperparameters
99
+
100
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
101
+
102
+ #### Speeds, Sizes, Times [optional]
103
+
104
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
105
+
106
+ [More Information Needed]
107
+
108
+ ## Evaluation
109
+
110
+ <!-- This section describes the evaluation protocols and provides the results. -->
111
+
112
+ ### Testing Data, Factors & Metrics
113
+
114
+ #### Testing Data
115
+
116
+ <!-- This should link to a Dataset Card if possible. -->
117
+
118
+ [More Information Needed]
119
+
120
+ #### Factors
121
+
122
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
123
+
124
+ [More Information Needed]
125
+
126
+ #### Metrics
127
+
128
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
129
+
130
+ [More Information Needed]
131
+
132
+ ### Results
133
+
134
+ [More Information Needed]
135
+
136
+ #### Summary
137
+
138
+
139
+
140
+ ## Model Examination [optional]
141
+
142
+ <!-- Relevant interpretability work for the model goes here -->
143
+
144
+ [More Information Needed]
145
+
146
+ ## Environmental Impact
147
+
148
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
149
+
150
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
151
+
152
+ - **Hardware Type:** [More Information Needed]
153
+ - **Hours used:** [More Information Needed]
154
+ - **Cloud Provider:** [More Information Needed]
155
+ - **Compute Region:** [More Information Needed]
156
+ - **Carbon Emitted:** [More Information Needed]
157
+
158
+ ## Technical Specifications [optional]
159
+
160
+ ### Model Architecture and Objective
161
+
162
+ [More Information Needed]
163
+
164
+ ### Compute Infrastructure
165
+
166
+ [More Information Needed]
167
+
168
+ #### Hardware
169
+
170
+ [More Information Needed]
171
+
172
+ #### Software
173
+
174
+ [More Information Needed]
175
+
176
+ ## Citation [optional]
177
+
178
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
179
+
180
+ **BibTeX:**
181
+
182
+ [More Information Needed]
183
+
184
+ **APA:**
185
+
186
+ [More Information Needed]
187
+
188
+ ## Glossary [optional]
189
+
190
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
191
+
192
+ [More Information Needed]
193
+
194
+ ## More Information [optional]
195
+
196
+ [More Information Needed]
197
+
198
+ ## Model Card Authors [optional]
199
+
200
+ [More Information Needed]
201
+
202
+ ## Model Card Contact
203
+
204
+ [More Information Needed]
205
+ ### Framework versions
206
+
207
+ - PEFT 0.16.0
smollm2-finetuned-lora1/adapter_config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "HuggingFaceTB/SmolLM2-360M",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 8,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "q_proj",
29
+ "v_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "trainable_token_indices": null,
33
+ "use_dora": false,
34
+ "use_qalora": false,
35
+ "use_rslora": false
36
+ }
smollm2-finetuned-lora1/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7435cc9d057e9ee49e9490c9d73d413da59ad25ccf7cf29c4dff151f4553d800
3
+ size 3293480
smollm2-finetuned-lora1/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned-lora1/special_tokens_map.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|im_start|>",
5
+ "<|im_end|>",
6
+ "<repo_name>",
7
+ "<reponame>",
8
+ "<file_sep>",
9
+ "<filename>",
10
+ "<gh_stars>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<jupyter_script>",
19
+ "<empty_output>"
20
+ ],
21
+ "bos_token": {
22
+ "content": "<|endoftext|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "eos_token": {
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "pad_token": "<|endoftext|>",
36
+ "unk_token": {
37
+ "content": "<|endoftext|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false
42
+ }
43
+ }
smollm2-finetuned-lora1/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned-lora1/tokenizer_config.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<repo_name>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<reponame>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<file_sep>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<filename>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<gh_stars>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<jupyter_script>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<empty_output>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ }
140
+ },
141
+ "additional_special_tokens": [
142
+ "<|endoftext|>",
143
+ "<|im_start|>",
144
+ "<|im_end|>",
145
+ "<repo_name>",
146
+ "<reponame>",
147
+ "<file_sep>",
148
+ "<filename>",
149
+ "<gh_stars>",
150
+ "<issue_start>",
151
+ "<issue_comment>",
152
+ "<issue_closed>",
153
+ "<jupyter_start>",
154
+ "<jupyter_text>",
155
+ "<jupyter_code>",
156
+ "<jupyter_output>",
157
+ "<jupyter_script>",
158
+ "<empty_output>"
159
+ ],
160
+ "bos_token": "<|endoftext|>",
161
+ "clean_up_tokenization_spaces": false,
162
+ "eos_token": "<|endoftext|>",
163
+ "extra_special_tokens": {},
164
+ "model_max_length": 8192,
165
+ "pad_token": "<|endoftext|>",
166
+ "tokenizer_class": "GPT2Tokenizer",
167
+ "unk_token": "<|endoftext|>",
168
+ "vocab_size": 49152
169
+ }
smollm2-finetuned-lora1/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-192/README.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: HuggingFaceTB/SmolLM2-360M
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:HuggingFaceTB/SmolLM2-360M
7
+ - lora
8
+ - transformers
9
+ ---
10
+
11
+ # Model Card for Model ID
12
+
13
+ <!-- Provide a quick summary of what the model is/does. -->
14
+
15
+
16
+
17
+ ## Model Details
18
+
19
+ ### Model Description
20
+
21
+ <!-- Provide a longer summary of what this model is. -->
22
+
23
+
24
+
25
+ - **Developed by:** [More Information Needed]
26
+ - **Funded by [optional]:** [More Information Needed]
27
+ - **Shared by [optional]:** [More Information Needed]
28
+ - **Model type:** [More Information Needed]
29
+ - **Language(s) (NLP):** [More Information Needed]
30
+ - **License:** [More Information Needed]
31
+ - **Finetuned from model [optional]:** [More Information Needed]
32
+
33
+ ### Model Sources [optional]
34
+
35
+ <!-- Provide the basic links for the model. -->
36
+
37
+ - **Repository:** [More Information Needed]
38
+ - **Paper [optional]:** [More Information Needed]
39
+ - **Demo [optional]:** [More Information Needed]
40
+
41
+ ## Uses
42
+
43
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
44
+
45
+ ### Direct Use
46
+
47
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
48
+
49
+ [More Information Needed]
50
+
51
+ ### Downstream Use [optional]
52
+
53
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
54
+
55
+ [More Information Needed]
56
+
57
+ ### Out-of-Scope Use
58
+
59
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
60
+
61
+ [More Information Needed]
62
+
63
+ ## Bias, Risks, and Limitations
64
+
65
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
66
+
67
+ [More Information Needed]
68
+
69
+ ### Recommendations
70
+
71
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
72
+
73
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
74
+
75
+ ## How to Get Started with the Model
76
+
77
+ Use the code below to get started with the model.
78
+
79
+ [More Information Needed]
80
+
81
+ ## Training Details
82
+
83
+ ### Training Data
84
+
85
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
86
+
87
+ [More Information Needed]
88
+
89
+ ### Training Procedure
90
+
91
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
92
+
93
+ #### Preprocessing [optional]
94
+
95
+ [More Information Needed]
96
+
97
+
98
+ #### Training Hyperparameters
99
+
100
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
101
+
102
+ #### Speeds, Sizes, Times [optional]
103
+
104
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
105
+
106
+ [More Information Needed]
107
+
108
+ ## Evaluation
109
+
110
+ <!-- This section describes the evaluation protocols and provides the results. -->
111
+
112
+ ### Testing Data, Factors & Metrics
113
+
114
+ #### Testing Data
115
+
116
+ <!-- This should link to a Dataset Card if possible. -->
117
+
118
+ [More Information Needed]
119
+
120
+ #### Factors
121
+
122
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
123
+
124
+ [More Information Needed]
125
+
126
+ #### Metrics
127
+
128
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
129
+
130
+ [More Information Needed]
131
+
132
+ ### Results
133
+
134
+ [More Information Needed]
135
+
136
+ #### Summary
137
+
138
+
139
+
140
+ ## Model Examination [optional]
141
+
142
+ <!-- Relevant interpretability work for the model goes here -->
143
+
144
+ [More Information Needed]
145
+
146
+ ## Environmental Impact
147
+
148
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
149
+
150
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
151
+
152
+ - **Hardware Type:** [More Information Needed]
153
+ - **Hours used:** [More Information Needed]
154
+ - **Cloud Provider:** [More Information Needed]
155
+ - **Compute Region:** [More Information Needed]
156
+ - **Carbon Emitted:** [More Information Needed]
157
+
158
+ ## Technical Specifications [optional]
159
+
160
+ ### Model Architecture and Objective
161
+
162
+ [More Information Needed]
163
+
164
+ ### Compute Infrastructure
165
+
166
+ [More Information Needed]
167
+
168
+ #### Hardware
169
+
170
+ [More Information Needed]
171
+
172
+ #### Software
173
+
174
+ [More Information Needed]
175
+
176
+ ## Citation [optional]
177
+
178
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
179
+
180
+ **BibTeX:**
181
+
182
+ [More Information Needed]
183
+
184
+ **APA:**
185
+
186
+ [More Information Needed]
187
+
188
+ ## Glossary [optional]
189
+
190
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
191
+
192
+ [More Information Needed]
193
+
194
+ ## More Information [optional]
195
+
196
+ [More Information Needed]
197
+
198
+ ## Model Card Authors [optional]
199
+
200
+ [More Information Needed]
201
+
202
+ ## Model Card Contact
203
+
204
+ [More Information Needed]
205
+ ### Framework versions
206
+
207
+ - PEFT 0.16.0
smollm2-finetuned1/checkpoint-192/adapter_config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "HuggingFaceTB/SmolLM2-360M",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 8,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "q_proj",
29
+ "v_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "trainable_token_indices": null,
33
+ "use_dora": false,
34
+ "use_qalora": false,
35
+ "use_rslora": false
36
+ }
smollm2-finetuned1/checkpoint-192/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4407b1f44d5355d9ed5519b75fca92438582321a2520ef8419f95337cd93489
3
+ size 3293480
smollm2-finetuned1/checkpoint-192/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-192/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3fe0032b83a1697667f78bccf2ffb9fd0a3b269c94be0c5799c0aa5312950c5
3
+ size 6661242
smollm2-finetuned1/checkpoint-192/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c22230a48bd012898193bfe3e0f6510e5b5002668323232e1c1c700ef9a92242
3
+ size 14244
smollm2-finetuned1/checkpoint-192/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c2fe68fc095d1d2c2e3b95d90b4e227595fb816cf55fdd03beb453c76dee2f5
3
+ size 988
smollm2-finetuned1/checkpoint-192/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:267aac5eee71f282e026c43a02ff27dbb79b56622a3f11114b9f2e481627b827
3
+ size 1064
smollm2-finetuned1/checkpoint-192/special_tokens_map.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|im_start|>",
5
+ "<|im_end|>",
6
+ "<repo_name>",
7
+ "<reponame>",
8
+ "<file_sep>",
9
+ "<filename>",
10
+ "<gh_stars>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<jupyter_script>",
19
+ "<empty_output>"
20
+ ],
21
+ "bos_token": {
22
+ "content": "<|endoftext|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "eos_token": {
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "pad_token": "<|endoftext|>",
36
+ "unk_token": {
37
+ "content": "<|endoftext|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false
42
+ }
43
+ }
smollm2-finetuned1/checkpoint-192/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-192/tokenizer_config.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<repo_name>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<reponame>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<file_sep>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<filename>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<gh_stars>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<jupyter_script>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<empty_output>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ }
140
+ },
141
+ "additional_special_tokens": [
142
+ "<|endoftext|>",
143
+ "<|im_start|>",
144
+ "<|im_end|>",
145
+ "<repo_name>",
146
+ "<reponame>",
147
+ "<file_sep>",
148
+ "<filename>",
149
+ "<gh_stars>",
150
+ "<issue_start>",
151
+ "<issue_comment>",
152
+ "<issue_closed>",
153
+ "<jupyter_start>",
154
+ "<jupyter_text>",
155
+ "<jupyter_code>",
156
+ "<jupyter_output>",
157
+ "<jupyter_script>",
158
+ "<empty_output>"
159
+ ],
160
+ "bos_token": "<|endoftext|>",
161
+ "clean_up_tokenization_spaces": false,
162
+ "eos_token": "<|endoftext|>",
163
+ "extra_special_tokens": {},
164
+ "model_max_length": 8192,
165
+ "pad_token": "<|endoftext|>",
166
+ "tokenizer_class": "GPT2Tokenizer",
167
+ "unk_token": "<|endoftext|>",
168
+ "vocab_size": 49152
169
+ }
smollm2-finetuned1/checkpoint-192/trainer_state.json ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.0,
6
+ "eval_steps": 500,
7
+ "global_step": 192,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.052083333333333336,
14
+ "grad_norm": 0.07655070722103119,
15
+ "learning_rate": 4.921875e-05,
16
+ "loss": 3.1871,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.10416666666666667,
21
+ "grad_norm": 0.09160477668046951,
22
+ "learning_rate": 4.835069444444444e-05,
23
+ "loss": 3.0592,
24
+ "step": 20
25
+ },
26
+ {
27
+ "epoch": 0.15625,
28
+ "grad_norm": 0.09234557300806046,
29
+ "learning_rate": 4.748263888888889e-05,
30
+ "loss": 3.1982,
31
+ "step": 30
32
+ },
33
+ {
34
+ "epoch": 0.20833333333333334,
35
+ "grad_norm": 0.09615059196949005,
36
+ "learning_rate": 4.6614583333333336e-05,
37
+ "loss": 3.1865,
38
+ "step": 40
39
+ },
40
+ {
41
+ "epoch": 0.2604166666666667,
42
+ "grad_norm": 0.11428328603506088,
43
+ "learning_rate": 4.574652777777778e-05,
44
+ "loss": 3.1689,
45
+ "step": 50
46
+ },
47
+ {
48
+ "epoch": 0.3125,
49
+ "grad_norm": 0.10111145675182343,
50
+ "learning_rate": 4.487847222222222e-05,
51
+ "loss": 3.0937,
52
+ "step": 60
53
+ },
54
+ {
55
+ "epoch": 0.3645833333333333,
56
+ "grad_norm": 0.12738557159900665,
57
+ "learning_rate": 4.401041666666667e-05,
58
+ "loss": 3.143,
59
+ "step": 70
60
+ },
61
+ {
62
+ "epoch": 0.4166666666666667,
63
+ "grad_norm": 0.14076603949069977,
64
+ "learning_rate": 4.3142361111111114e-05,
65
+ "loss": 3.0963,
66
+ "step": 80
67
+ },
68
+ {
69
+ "epoch": 0.46875,
70
+ "grad_norm": 0.12179583311080933,
71
+ "learning_rate": 4.227430555555556e-05,
72
+ "loss": 3.0358,
73
+ "step": 90
74
+ },
75
+ {
76
+ "epoch": 0.5208333333333334,
77
+ "grad_norm": 0.13192890584468842,
78
+ "learning_rate": 4.140625e-05,
79
+ "loss": 3.1517,
80
+ "step": 100
81
+ },
82
+ {
83
+ "epoch": 0.5729166666666666,
84
+ "grad_norm": 0.12306984513998032,
85
+ "learning_rate": 4.053819444444444e-05,
86
+ "loss": 3.0126,
87
+ "step": 110
88
+ },
89
+ {
90
+ "epoch": 0.625,
91
+ "grad_norm": 0.1288878321647644,
92
+ "learning_rate": 3.967013888888889e-05,
93
+ "loss": 3.0296,
94
+ "step": 120
95
+ },
96
+ {
97
+ "epoch": 0.6770833333333334,
98
+ "grad_norm": 0.15506671369075775,
99
+ "learning_rate": 3.8802083333333336e-05,
100
+ "loss": 3.0453,
101
+ "step": 130
102
+ },
103
+ {
104
+ "epoch": 0.7291666666666666,
105
+ "grad_norm": 0.14591804146766663,
106
+ "learning_rate": 3.793402777777778e-05,
107
+ "loss": 3.0611,
108
+ "step": 140
109
+ },
110
+ {
111
+ "epoch": 0.78125,
112
+ "grad_norm": 0.14732398092746735,
113
+ "learning_rate": 3.706597222222222e-05,
114
+ "loss": 3.071,
115
+ "step": 150
116
+ },
117
+ {
118
+ "epoch": 0.8333333333333334,
119
+ "grad_norm": 0.14069262146949768,
120
+ "learning_rate": 3.619791666666667e-05,
121
+ "loss": 3.0066,
122
+ "step": 160
123
+ },
124
+ {
125
+ "epoch": 0.8854166666666666,
126
+ "grad_norm": 0.15924516320228577,
127
+ "learning_rate": 3.5329861111111114e-05,
128
+ "loss": 2.9758,
129
+ "step": 170
130
+ },
131
+ {
132
+ "epoch": 0.9375,
133
+ "grad_norm": 0.16019809246063232,
134
+ "learning_rate": 3.446180555555556e-05,
135
+ "loss": 2.981,
136
+ "step": 180
137
+ },
138
+ {
139
+ "epoch": 0.9895833333333334,
140
+ "grad_norm": 0.155264750123024,
141
+ "learning_rate": 3.359375e-05,
142
+ "loss": 2.9057,
143
+ "step": 190
144
+ }
145
+ ],
146
+ "logging_steps": 10,
147
+ "max_steps": 576,
148
+ "num_input_tokens_seen": 0,
149
+ "num_train_epochs": 3,
150
+ "save_steps": 500,
151
+ "stateful_callbacks": {
152
+ "TrainerControl": {
153
+ "args": {
154
+ "should_epoch_stop": false,
155
+ "should_evaluate": false,
156
+ "should_log": false,
157
+ "should_save": true,
158
+ "should_training_stop": false
159
+ },
160
+ "attributes": {}
161
+ }
162
+ },
163
+ "total_flos": 1488500608204800.0,
164
+ "train_batch_size": 4,
165
+ "trial_name": null,
166
+ "trial_params": null
167
+ }
smollm2-finetuned1/checkpoint-192/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42bc3678940440ddc35e42d91a045b6aaec8a51f1e35336d47713fe931405bab
3
+ size 5304
smollm2-finetuned1/checkpoint-192/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-384/README.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: HuggingFaceTB/SmolLM2-360M
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:HuggingFaceTB/SmolLM2-360M
7
+ - lora
8
+ - transformers
9
+ ---
10
+
11
+ # Model Card for Model ID
12
+
13
+ <!-- Provide a quick summary of what the model is/does. -->
14
+
15
+
16
+
17
+ ## Model Details
18
+
19
+ ### Model Description
20
+
21
+ <!-- Provide a longer summary of what this model is. -->
22
+
23
+
24
+
25
+ - **Developed by:** [More Information Needed]
26
+ - **Funded by [optional]:** [More Information Needed]
27
+ - **Shared by [optional]:** [More Information Needed]
28
+ - **Model type:** [More Information Needed]
29
+ - **Language(s) (NLP):** [More Information Needed]
30
+ - **License:** [More Information Needed]
31
+ - **Finetuned from model [optional]:** [More Information Needed]
32
+
33
+ ### Model Sources [optional]
34
+
35
+ <!-- Provide the basic links for the model. -->
36
+
37
+ - **Repository:** [More Information Needed]
38
+ - **Paper [optional]:** [More Information Needed]
39
+ - **Demo [optional]:** [More Information Needed]
40
+
41
+ ## Uses
42
+
43
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
44
+
45
+ ### Direct Use
46
+
47
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
48
+
49
+ [More Information Needed]
50
+
51
+ ### Downstream Use [optional]
52
+
53
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
54
+
55
+ [More Information Needed]
56
+
57
+ ### Out-of-Scope Use
58
+
59
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
60
+
61
+ [More Information Needed]
62
+
63
+ ## Bias, Risks, and Limitations
64
+
65
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
66
+
67
+ [More Information Needed]
68
+
69
+ ### Recommendations
70
+
71
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
72
+
73
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
74
+
75
+ ## How to Get Started with the Model
76
+
77
+ Use the code below to get started with the model.
78
+
79
+ [More Information Needed]
80
+
81
+ ## Training Details
82
+
83
+ ### Training Data
84
+
85
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
86
+
87
+ [More Information Needed]
88
+
89
+ ### Training Procedure
90
+
91
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
92
+
93
+ #### Preprocessing [optional]
94
+
95
+ [More Information Needed]
96
+
97
+
98
+ #### Training Hyperparameters
99
+
100
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
101
+
102
+ #### Speeds, Sizes, Times [optional]
103
+
104
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
105
+
106
+ [More Information Needed]
107
+
108
+ ## Evaluation
109
+
110
+ <!-- This section describes the evaluation protocols and provides the results. -->
111
+
112
+ ### Testing Data, Factors & Metrics
113
+
114
+ #### Testing Data
115
+
116
+ <!-- This should link to a Dataset Card if possible. -->
117
+
118
+ [More Information Needed]
119
+
120
+ #### Factors
121
+
122
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
123
+
124
+ [More Information Needed]
125
+
126
+ #### Metrics
127
+
128
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
129
+
130
+ [More Information Needed]
131
+
132
+ ### Results
133
+
134
+ [More Information Needed]
135
+
136
+ #### Summary
137
+
138
+
139
+
140
+ ## Model Examination [optional]
141
+
142
+ <!-- Relevant interpretability work for the model goes here -->
143
+
144
+ [More Information Needed]
145
+
146
+ ## Environmental Impact
147
+
148
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
149
+
150
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
151
+
152
+ - **Hardware Type:** [More Information Needed]
153
+ - **Hours used:** [More Information Needed]
154
+ - **Cloud Provider:** [More Information Needed]
155
+ - **Compute Region:** [More Information Needed]
156
+ - **Carbon Emitted:** [More Information Needed]
157
+
158
+ ## Technical Specifications [optional]
159
+
160
+ ### Model Architecture and Objective
161
+
162
+ [More Information Needed]
163
+
164
+ ### Compute Infrastructure
165
+
166
+ [More Information Needed]
167
+
168
+ #### Hardware
169
+
170
+ [More Information Needed]
171
+
172
+ #### Software
173
+
174
+ [More Information Needed]
175
+
176
+ ## Citation [optional]
177
+
178
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
179
+
180
+ **BibTeX:**
181
+
182
+ [More Information Needed]
183
+
184
+ **APA:**
185
+
186
+ [More Information Needed]
187
+
188
+ ## Glossary [optional]
189
+
190
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
191
+
192
+ [More Information Needed]
193
+
194
+ ## More Information [optional]
195
+
196
+ [More Information Needed]
197
+
198
+ ## Model Card Authors [optional]
199
+
200
+ [More Information Needed]
201
+
202
+ ## Model Card Contact
203
+
204
+ [More Information Needed]
205
+ ### Framework versions
206
+
207
+ - PEFT 0.16.0
smollm2-finetuned1/checkpoint-384/adapter_config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "HuggingFaceTB/SmolLM2-360M",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 8,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "q_proj",
29
+ "v_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "trainable_token_indices": null,
33
+ "use_dora": false,
34
+ "use_qalora": false,
35
+ "use_rslora": false
36
+ }
smollm2-finetuned1/checkpoint-384/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bf151157d1f76e14f52e8de20b9e7d775ce94fec46cd9589f75b8f0805293dd
3
+ size 3293480
smollm2-finetuned1/checkpoint-384/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-384/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bd86b123fed0dee5276a82fa1d2817f7342188e56060b1c9c2481f26e623ef2
3
+ size 6661242
smollm2-finetuned1/checkpoint-384/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b02962bcb741e7aee4061cc720d2d3dca2b4a932ac83ffb3fbb613b2231eaaf7
3
+ size 14244
smollm2-finetuned1/checkpoint-384/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95aa9831d938a08de46259e8bce5e71c5a827f2fb143f76ce2849bb1e27b78a9
3
+ size 988
smollm2-finetuned1/checkpoint-384/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45dc0633de6c44e0714dfe6fe328cb8bf2f34f32c97bc8b6eac54e156ea5622d
3
+ size 1064
smollm2-finetuned1/checkpoint-384/special_tokens_map.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|im_start|>",
5
+ "<|im_end|>",
6
+ "<repo_name>",
7
+ "<reponame>",
8
+ "<file_sep>",
9
+ "<filename>",
10
+ "<gh_stars>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<jupyter_script>",
19
+ "<empty_output>"
20
+ ],
21
+ "bos_token": {
22
+ "content": "<|endoftext|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "eos_token": {
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "pad_token": "<|endoftext|>",
36
+ "unk_token": {
37
+ "content": "<|endoftext|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false
42
+ }
43
+ }
smollm2-finetuned1/checkpoint-384/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-384/tokenizer_config.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<repo_name>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<reponame>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<file_sep>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<filename>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<gh_stars>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<jupyter_script>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<empty_output>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ }
140
+ },
141
+ "additional_special_tokens": [
142
+ "<|endoftext|>",
143
+ "<|im_start|>",
144
+ "<|im_end|>",
145
+ "<repo_name>",
146
+ "<reponame>",
147
+ "<file_sep>",
148
+ "<filename>",
149
+ "<gh_stars>",
150
+ "<issue_start>",
151
+ "<issue_comment>",
152
+ "<issue_closed>",
153
+ "<jupyter_start>",
154
+ "<jupyter_text>",
155
+ "<jupyter_code>",
156
+ "<jupyter_output>",
157
+ "<jupyter_script>",
158
+ "<empty_output>"
159
+ ],
160
+ "bos_token": "<|endoftext|>",
161
+ "clean_up_tokenization_spaces": false,
162
+ "eos_token": "<|endoftext|>",
163
+ "extra_special_tokens": {},
164
+ "model_max_length": 8192,
165
+ "pad_token": "<|endoftext|>",
166
+ "tokenizer_class": "GPT2Tokenizer",
167
+ "unk_token": "<|endoftext|>",
168
+ "vocab_size": 49152
169
+ }
smollm2-finetuned1/checkpoint-384/trainer_state.json ADDED
@@ -0,0 +1,300 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 2.0,
6
+ "eval_steps": 500,
7
+ "global_step": 384,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.052083333333333336,
14
+ "grad_norm": 0.07655070722103119,
15
+ "learning_rate": 4.921875e-05,
16
+ "loss": 3.1871,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.10416666666666667,
21
+ "grad_norm": 0.09160477668046951,
22
+ "learning_rate": 4.835069444444444e-05,
23
+ "loss": 3.0592,
24
+ "step": 20
25
+ },
26
+ {
27
+ "epoch": 0.15625,
28
+ "grad_norm": 0.09234557300806046,
29
+ "learning_rate": 4.748263888888889e-05,
30
+ "loss": 3.1982,
31
+ "step": 30
32
+ },
33
+ {
34
+ "epoch": 0.20833333333333334,
35
+ "grad_norm": 0.09615059196949005,
36
+ "learning_rate": 4.6614583333333336e-05,
37
+ "loss": 3.1865,
38
+ "step": 40
39
+ },
40
+ {
41
+ "epoch": 0.2604166666666667,
42
+ "grad_norm": 0.11428328603506088,
43
+ "learning_rate": 4.574652777777778e-05,
44
+ "loss": 3.1689,
45
+ "step": 50
46
+ },
47
+ {
48
+ "epoch": 0.3125,
49
+ "grad_norm": 0.10111145675182343,
50
+ "learning_rate": 4.487847222222222e-05,
51
+ "loss": 3.0937,
52
+ "step": 60
53
+ },
54
+ {
55
+ "epoch": 0.3645833333333333,
56
+ "grad_norm": 0.12738557159900665,
57
+ "learning_rate": 4.401041666666667e-05,
58
+ "loss": 3.143,
59
+ "step": 70
60
+ },
61
+ {
62
+ "epoch": 0.4166666666666667,
63
+ "grad_norm": 0.14076603949069977,
64
+ "learning_rate": 4.3142361111111114e-05,
65
+ "loss": 3.0963,
66
+ "step": 80
67
+ },
68
+ {
69
+ "epoch": 0.46875,
70
+ "grad_norm": 0.12179583311080933,
71
+ "learning_rate": 4.227430555555556e-05,
72
+ "loss": 3.0358,
73
+ "step": 90
74
+ },
75
+ {
76
+ "epoch": 0.5208333333333334,
77
+ "grad_norm": 0.13192890584468842,
78
+ "learning_rate": 4.140625e-05,
79
+ "loss": 3.1517,
80
+ "step": 100
81
+ },
82
+ {
83
+ "epoch": 0.5729166666666666,
84
+ "grad_norm": 0.12306984513998032,
85
+ "learning_rate": 4.053819444444444e-05,
86
+ "loss": 3.0126,
87
+ "step": 110
88
+ },
89
+ {
90
+ "epoch": 0.625,
91
+ "grad_norm": 0.1288878321647644,
92
+ "learning_rate": 3.967013888888889e-05,
93
+ "loss": 3.0296,
94
+ "step": 120
95
+ },
96
+ {
97
+ "epoch": 0.6770833333333334,
98
+ "grad_norm": 0.15506671369075775,
99
+ "learning_rate": 3.8802083333333336e-05,
100
+ "loss": 3.0453,
101
+ "step": 130
102
+ },
103
+ {
104
+ "epoch": 0.7291666666666666,
105
+ "grad_norm": 0.14591804146766663,
106
+ "learning_rate": 3.793402777777778e-05,
107
+ "loss": 3.0611,
108
+ "step": 140
109
+ },
110
+ {
111
+ "epoch": 0.78125,
112
+ "grad_norm": 0.14732398092746735,
113
+ "learning_rate": 3.706597222222222e-05,
114
+ "loss": 3.071,
115
+ "step": 150
116
+ },
117
+ {
118
+ "epoch": 0.8333333333333334,
119
+ "grad_norm": 0.14069262146949768,
120
+ "learning_rate": 3.619791666666667e-05,
121
+ "loss": 3.0066,
122
+ "step": 160
123
+ },
124
+ {
125
+ "epoch": 0.8854166666666666,
126
+ "grad_norm": 0.15924516320228577,
127
+ "learning_rate": 3.5329861111111114e-05,
128
+ "loss": 2.9758,
129
+ "step": 170
130
+ },
131
+ {
132
+ "epoch": 0.9375,
133
+ "grad_norm": 0.16019809246063232,
134
+ "learning_rate": 3.446180555555556e-05,
135
+ "loss": 2.981,
136
+ "step": 180
137
+ },
138
+ {
139
+ "epoch": 0.9895833333333334,
140
+ "grad_norm": 0.155264750123024,
141
+ "learning_rate": 3.359375e-05,
142
+ "loss": 2.9057,
143
+ "step": 190
144
+ },
145
+ {
146
+ "epoch": 1.0416666666666667,
147
+ "grad_norm": 0.1712893545627594,
148
+ "learning_rate": 3.272569444444444e-05,
149
+ "loss": 2.9711,
150
+ "step": 200
151
+ },
152
+ {
153
+ "epoch": 1.09375,
154
+ "grad_norm": 0.1758311241865158,
155
+ "learning_rate": 3.185763888888889e-05,
156
+ "loss": 2.9298,
157
+ "step": 210
158
+ },
159
+ {
160
+ "epoch": 1.1458333333333333,
161
+ "grad_norm": 0.17173148691654205,
162
+ "learning_rate": 3.0989583333333336e-05,
163
+ "loss": 2.9685,
164
+ "step": 220
165
+ },
166
+ {
167
+ "epoch": 1.1979166666666667,
168
+ "grad_norm": 0.19112636148929596,
169
+ "learning_rate": 3.0121527777777782e-05,
170
+ "loss": 2.9262,
171
+ "step": 230
172
+ },
173
+ {
174
+ "epoch": 1.25,
175
+ "grad_norm": 0.18498270213603973,
176
+ "learning_rate": 2.925347222222222e-05,
177
+ "loss": 2.8788,
178
+ "step": 240
179
+ },
180
+ {
181
+ "epoch": 1.3020833333333333,
182
+ "grad_norm": 0.1705554723739624,
183
+ "learning_rate": 2.838541666666667e-05,
184
+ "loss": 2.7591,
185
+ "step": 250
186
+ },
187
+ {
188
+ "epoch": 1.3541666666666667,
189
+ "grad_norm": 0.19246017932891846,
190
+ "learning_rate": 2.751736111111111e-05,
191
+ "loss": 2.8625,
192
+ "step": 260
193
+ },
194
+ {
195
+ "epoch": 1.40625,
196
+ "grad_norm": 0.17858931422233582,
197
+ "learning_rate": 2.664930555555556e-05,
198
+ "loss": 2.9046,
199
+ "step": 270
200
+ },
201
+ {
202
+ "epoch": 1.4583333333333333,
203
+ "grad_norm": 0.21472914516925812,
204
+ "learning_rate": 2.578125e-05,
205
+ "loss": 2.8826,
206
+ "step": 280
207
+ },
208
+ {
209
+ "epoch": 1.5104166666666665,
210
+ "grad_norm": 0.18860450387001038,
211
+ "learning_rate": 2.4913194444444446e-05,
212
+ "loss": 2.8265,
213
+ "step": 290
214
+ },
215
+ {
216
+ "epoch": 1.5625,
217
+ "grad_norm": 0.17438200116157532,
218
+ "learning_rate": 2.404513888888889e-05,
219
+ "loss": 2.7819,
220
+ "step": 300
221
+ },
222
+ {
223
+ "epoch": 1.6145833333333335,
224
+ "grad_norm": 0.23125068843364716,
225
+ "learning_rate": 2.3177083333333335e-05,
226
+ "loss": 2.8364,
227
+ "step": 310
228
+ },
229
+ {
230
+ "epoch": 1.6666666666666665,
231
+ "grad_norm": 0.18903665244579315,
232
+ "learning_rate": 2.2309027777777782e-05,
233
+ "loss": 2.8423,
234
+ "step": 320
235
+ },
236
+ {
237
+ "epoch": 1.71875,
238
+ "grad_norm": 0.22534161806106567,
239
+ "learning_rate": 2.144097222222222e-05,
240
+ "loss": 2.8666,
241
+ "step": 330
242
+ },
243
+ {
244
+ "epoch": 1.7708333333333335,
245
+ "grad_norm": 0.240483358502388,
246
+ "learning_rate": 2.0572916666666668e-05,
247
+ "loss": 2.8409,
248
+ "step": 340
249
+ },
250
+ {
251
+ "epoch": 1.8229166666666665,
252
+ "grad_norm": 0.23081159591674805,
253
+ "learning_rate": 1.970486111111111e-05,
254
+ "loss": 2.8257,
255
+ "step": 350
256
+ },
257
+ {
258
+ "epoch": 1.875,
259
+ "grad_norm": 0.20717695355415344,
260
+ "learning_rate": 1.8836805555555557e-05,
261
+ "loss": 2.9202,
262
+ "step": 360
263
+ },
264
+ {
265
+ "epoch": 1.9270833333333335,
266
+ "grad_norm": 0.21534092724323273,
267
+ "learning_rate": 1.796875e-05,
268
+ "loss": 2.7127,
269
+ "step": 370
270
+ },
271
+ {
272
+ "epoch": 1.9791666666666665,
273
+ "grad_norm": 0.24049048125743866,
274
+ "learning_rate": 1.7100694444444446e-05,
275
+ "loss": 2.8734,
276
+ "step": 380
277
+ }
278
+ ],
279
+ "logging_steps": 10,
280
+ "max_steps": 576,
281
+ "num_input_tokens_seen": 0,
282
+ "num_train_epochs": 3,
283
+ "save_steps": 500,
284
+ "stateful_callbacks": {
285
+ "TrainerControl": {
286
+ "args": {
287
+ "should_epoch_stop": false,
288
+ "should_evaluate": false,
289
+ "should_log": false,
290
+ "should_save": true,
291
+ "should_training_stop": false
292
+ },
293
+ "attributes": {}
294
+ }
295
+ },
296
+ "total_flos": 2977001216409600.0,
297
+ "train_batch_size": 4,
298
+ "trial_name": null,
299
+ "trial_params": null
300
+ }
smollm2-finetuned1/checkpoint-384/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42bc3678940440ddc35e42d91a045b6aaec8a51f1e35336d47713fe931405bab
3
+ size 5304
smollm2-finetuned1/checkpoint-384/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-576/README.md ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: HuggingFaceTB/SmolLM2-360M
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - base_model:adapter:HuggingFaceTB/SmolLM2-360M
7
+ - lora
8
+ - transformers
9
+ ---
10
+
11
+ # Model Card for Model ID
12
+
13
+ <!-- Provide a quick summary of what the model is/does. -->
14
+
15
+
16
+
17
+ ## Model Details
18
+
19
+ ### Model Description
20
+
21
+ <!-- Provide a longer summary of what this model is. -->
22
+
23
+
24
+
25
+ - **Developed by:** [More Information Needed]
26
+ - **Funded by [optional]:** [More Information Needed]
27
+ - **Shared by [optional]:** [More Information Needed]
28
+ - **Model type:** [More Information Needed]
29
+ - **Language(s) (NLP):** [More Information Needed]
30
+ - **License:** [More Information Needed]
31
+ - **Finetuned from model [optional]:** [More Information Needed]
32
+
33
+ ### Model Sources [optional]
34
+
35
+ <!-- Provide the basic links for the model. -->
36
+
37
+ - **Repository:** [More Information Needed]
38
+ - **Paper [optional]:** [More Information Needed]
39
+ - **Demo [optional]:** [More Information Needed]
40
+
41
+ ## Uses
42
+
43
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
44
+
45
+ ### Direct Use
46
+
47
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
48
+
49
+ [More Information Needed]
50
+
51
+ ### Downstream Use [optional]
52
+
53
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
54
+
55
+ [More Information Needed]
56
+
57
+ ### Out-of-Scope Use
58
+
59
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
60
+
61
+ [More Information Needed]
62
+
63
+ ## Bias, Risks, and Limitations
64
+
65
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
66
+
67
+ [More Information Needed]
68
+
69
+ ### Recommendations
70
+
71
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
72
+
73
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
74
+
75
+ ## How to Get Started with the Model
76
+
77
+ Use the code below to get started with the model.
78
+
79
+ [More Information Needed]
80
+
81
+ ## Training Details
82
+
83
+ ### Training Data
84
+
85
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
86
+
87
+ [More Information Needed]
88
+
89
+ ### Training Procedure
90
+
91
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
92
+
93
+ #### Preprocessing [optional]
94
+
95
+ [More Information Needed]
96
+
97
+
98
+ #### Training Hyperparameters
99
+
100
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
101
+
102
+ #### Speeds, Sizes, Times [optional]
103
+
104
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
105
+
106
+ [More Information Needed]
107
+
108
+ ## Evaluation
109
+
110
+ <!-- This section describes the evaluation protocols and provides the results. -->
111
+
112
+ ### Testing Data, Factors & Metrics
113
+
114
+ #### Testing Data
115
+
116
+ <!-- This should link to a Dataset Card if possible. -->
117
+
118
+ [More Information Needed]
119
+
120
+ #### Factors
121
+
122
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
123
+
124
+ [More Information Needed]
125
+
126
+ #### Metrics
127
+
128
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
129
+
130
+ [More Information Needed]
131
+
132
+ ### Results
133
+
134
+ [More Information Needed]
135
+
136
+ #### Summary
137
+
138
+
139
+
140
+ ## Model Examination [optional]
141
+
142
+ <!-- Relevant interpretability work for the model goes here -->
143
+
144
+ [More Information Needed]
145
+
146
+ ## Environmental Impact
147
+
148
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
149
+
150
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
151
+
152
+ - **Hardware Type:** [More Information Needed]
153
+ - **Hours used:** [More Information Needed]
154
+ - **Cloud Provider:** [More Information Needed]
155
+ - **Compute Region:** [More Information Needed]
156
+ - **Carbon Emitted:** [More Information Needed]
157
+
158
+ ## Technical Specifications [optional]
159
+
160
+ ### Model Architecture and Objective
161
+
162
+ [More Information Needed]
163
+
164
+ ### Compute Infrastructure
165
+
166
+ [More Information Needed]
167
+
168
+ #### Hardware
169
+
170
+ [More Information Needed]
171
+
172
+ #### Software
173
+
174
+ [More Information Needed]
175
+
176
+ ## Citation [optional]
177
+
178
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
179
+
180
+ **BibTeX:**
181
+
182
+ [More Information Needed]
183
+
184
+ **APA:**
185
+
186
+ [More Information Needed]
187
+
188
+ ## Glossary [optional]
189
+
190
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
191
+
192
+ [More Information Needed]
193
+
194
+ ## More Information [optional]
195
+
196
+ [More Information Needed]
197
+
198
+ ## Model Card Authors [optional]
199
+
200
+ [More Information Needed]
201
+
202
+ ## Model Card Contact
203
+
204
+ [More Information Needed]
205
+ ### Framework versions
206
+
207
+ - PEFT 0.16.0
smollm2-finetuned1/checkpoint-576/adapter_config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "HuggingFaceTB/SmolLM2-360M",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 8,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "q_proj",
29
+ "v_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "trainable_token_indices": null,
33
+ "use_dora": false,
34
+ "use_qalora": false,
35
+ "use_rslora": false
36
+ }
smollm2-finetuned1/checkpoint-576/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7435cc9d057e9ee49e9490c9d73d413da59ad25ccf7cf29c4dff151f4553d800
3
+ size 3293480
smollm2-finetuned1/checkpoint-576/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-576/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46080e6ae59a4f79545166d471d7d3ee00b832beea5d6860b2f06b28411b4e5d
3
+ size 6661242
smollm2-finetuned1/checkpoint-576/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f8d8a426fd258fbd07bed2f99b07d5943b17c0d38975b4b6b9d92db68be111d
3
+ size 14244
smollm2-finetuned1/checkpoint-576/scaler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0738d3f7bc7a3d09895ccd1699afea3beb6076cacca6285d9dff68b3826864fb
3
+ size 988
smollm2-finetuned1/checkpoint-576/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5b8f918dc31b9ea1fc984479ad58621bac3235aad3af36583f5b48e928c8b91
3
+ size 1064
smollm2-finetuned1/checkpoint-576/special_tokens_map.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|im_start|>",
5
+ "<|im_end|>",
6
+ "<repo_name>",
7
+ "<reponame>",
8
+ "<file_sep>",
9
+ "<filename>",
10
+ "<gh_stars>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<jupyter_script>",
19
+ "<empty_output>"
20
+ ],
21
+ "bos_token": {
22
+ "content": "<|endoftext|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "eos_token": {
29
+ "content": "<|endoftext|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ },
35
+ "pad_token": "<|endoftext|>",
36
+ "unk_token": {
37
+ "content": "<|endoftext|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false
42
+ }
43
+ }
smollm2-finetuned1/checkpoint-576/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
smollm2-finetuned1/checkpoint-576/tokenizer_config.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<repo_name>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<reponame>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<file_sep>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<filename>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<gh_stars>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<jupyter_script>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<empty_output>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ }
140
+ },
141
+ "additional_special_tokens": [
142
+ "<|endoftext|>",
143
+ "<|im_start|>",
144
+ "<|im_end|>",
145
+ "<repo_name>",
146
+ "<reponame>",
147
+ "<file_sep>",
148
+ "<filename>",
149
+ "<gh_stars>",
150
+ "<issue_start>",
151
+ "<issue_comment>",
152
+ "<issue_closed>",
153
+ "<jupyter_start>",
154
+ "<jupyter_text>",
155
+ "<jupyter_code>",
156
+ "<jupyter_output>",
157
+ "<jupyter_script>",
158
+ "<empty_output>"
159
+ ],
160
+ "bos_token": "<|endoftext|>",
161
+ "clean_up_tokenization_spaces": false,
162
+ "eos_token": "<|endoftext|>",
163
+ "extra_special_tokens": {},
164
+ "model_max_length": 8192,
165
+ "pad_token": "<|endoftext|>",
166
+ "tokenizer_class": "GPT2Tokenizer",
167
+ "unk_token": "<|endoftext|>",
168
+ "vocab_size": 49152
169
+ }
smollm2-finetuned1/checkpoint-576/trainer_state.json ADDED
@@ -0,0 +1,433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 3.0,
6
+ "eval_steps": 500,
7
+ "global_step": 576,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.052083333333333336,
14
+ "grad_norm": 0.07655070722103119,
15
+ "learning_rate": 4.921875e-05,
16
+ "loss": 3.1871,
17
+ "step": 10
18
+ },
19
+ {
20
+ "epoch": 0.10416666666666667,
21
+ "grad_norm": 0.09160477668046951,
22
+ "learning_rate": 4.835069444444444e-05,
23
+ "loss": 3.0592,
24
+ "step": 20
25
+ },
26
+ {
27
+ "epoch": 0.15625,
28
+ "grad_norm": 0.09234557300806046,
29
+ "learning_rate": 4.748263888888889e-05,
30
+ "loss": 3.1982,
31
+ "step": 30
32
+ },
33
+ {
34
+ "epoch": 0.20833333333333334,
35
+ "grad_norm": 0.09615059196949005,
36
+ "learning_rate": 4.6614583333333336e-05,
37
+ "loss": 3.1865,
38
+ "step": 40
39
+ },
40
+ {
41
+ "epoch": 0.2604166666666667,
42
+ "grad_norm": 0.11428328603506088,
43
+ "learning_rate": 4.574652777777778e-05,
44
+ "loss": 3.1689,
45
+ "step": 50
46
+ },
47
+ {
48
+ "epoch": 0.3125,
49
+ "grad_norm": 0.10111145675182343,
50
+ "learning_rate": 4.487847222222222e-05,
51
+ "loss": 3.0937,
52
+ "step": 60
53
+ },
54
+ {
55
+ "epoch": 0.3645833333333333,
56
+ "grad_norm": 0.12738557159900665,
57
+ "learning_rate": 4.401041666666667e-05,
58
+ "loss": 3.143,
59
+ "step": 70
60
+ },
61
+ {
62
+ "epoch": 0.4166666666666667,
63
+ "grad_norm": 0.14076603949069977,
64
+ "learning_rate": 4.3142361111111114e-05,
65
+ "loss": 3.0963,
66
+ "step": 80
67
+ },
68
+ {
69
+ "epoch": 0.46875,
70
+ "grad_norm": 0.12179583311080933,
71
+ "learning_rate": 4.227430555555556e-05,
72
+ "loss": 3.0358,
73
+ "step": 90
74
+ },
75
+ {
76
+ "epoch": 0.5208333333333334,
77
+ "grad_norm": 0.13192890584468842,
78
+ "learning_rate": 4.140625e-05,
79
+ "loss": 3.1517,
80
+ "step": 100
81
+ },
82
+ {
83
+ "epoch": 0.5729166666666666,
84
+ "grad_norm": 0.12306984513998032,
85
+ "learning_rate": 4.053819444444444e-05,
86
+ "loss": 3.0126,
87
+ "step": 110
88
+ },
89
+ {
90
+ "epoch": 0.625,
91
+ "grad_norm": 0.1288878321647644,
92
+ "learning_rate": 3.967013888888889e-05,
93
+ "loss": 3.0296,
94
+ "step": 120
95
+ },
96
+ {
97
+ "epoch": 0.6770833333333334,
98
+ "grad_norm": 0.15506671369075775,
99
+ "learning_rate": 3.8802083333333336e-05,
100
+ "loss": 3.0453,
101
+ "step": 130
102
+ },
103
+ {
104
+ "epoch": 0.7291666666666666,
105
+ "grad_norm": 0.14591804146766663,
106
+ "learning_rate": 3.793402777777778e-05,
107
+ "loss": 3.0611,
108
+ "step": 140
109
+ },
110
+ {
111
+ "epoch": 0.78125,
112
+ "grad_norm": 0.14732398092746735,
113
+ "learning_rate": 3.706597222222222e-05,
114
+ "loss": 3.071,
115
+ "step": 150
116
+ },
117
+ {
118
+ "epoch": 0.8333333333333334,
119
+ "grad_norm": 0.14069262146949768,
120
+ "learning_rate": 3.619791666666667e-05,
121
+ "loss": 3.0066,
122
+ "step": 160
123
+ },
124
+ {
125
+ "epoch": 0.8854166666666666,
126
+ "grad_norm": 0.15924516320228577,
127
+ "learning_rate": 3.5329861111111114e-05,
128
+ "loss": 2.9758,
129
+ "step": 170
130
+ },
131
+ {
132
+ "epoch": 0.9375,
133
+ "grad_norm": 0.16019809246063232,
134
+ "learning_rate": 3.446180555555556e-05,
135
+ "loss": 2.981,
136
+ "step": 180
137
+ },
138
+ {
139
+ "epoch": 0.9895833333333334,
140
+ "grad_norm": 0.155264750123024,
141
+ "learning_rate": 3.359375e-05,
142
+ "loss": 2.9057,
143
+ "step": 190
144
+ },
145
+ {
146
+ "epoch": 1.0416666666666667,
147
+ "grad_norm": 0.1712893545627594,
148
+ "learning_rate": 3.272569444444444e-05,
149
+ "loss": 2.9711,
150
+ "step": 200
151
+ },
152
+ {
153
+ "epoch": 1.09375,
154
+ "grad_norm": 0.1758311241865158,
155
+ "learning_rate": 3.185763888888889e-05,
156
+ "loss": 2.9298,
157
+ "step": 210
158
+ },
159
+ {
160
+ "epoch": 1.1458333333333333,
161
+ "grad_norm": 0.17173148691654205,
162
+ "learning_rate": 3.0989583333333336e-05,
163
+ "loss": 2.9685,
164
+ "step": 220
165
+ },
166
+ {
167
+ "epoch": 1.1979166666666667,
168
+ "grad_norm": 0.19112636148929596,
169
+ "learning_rate": 3.0121527777777782e-05,
170
+ "loss": 2.9262,
171
+ "step": 230
172
+ },
173
+ {
174
+ "epoch": 1.25,
175
+ "grad_norm": 0.18498270213603973,
176
+ "learning_rate": 2.925347222222222e-05,
177
+ "loss": 2.8788,
178
+ "step": 240
179
+ },
180
+ {
181
+ "epoch": 1.3020833333333333,
182
+ "grad_norm": 0.1705554723739624,
183
+ "learning_rate": 2.838541666666667e-05,
184
+ "loss": 2.7591,
185
+ "step": 250
186
+ },
187
+ {
188
+ "epoch": 1.3541666666666667,
189
+ "grad_norm": 0.19246017932891846,
190
+ "learning_rate": 2.751736111111111e-05,
191
+ "loss": 2.8625,
192
+ "step": 260
193
+ },
194
+ {
195
+ "epoch": 1.40625,
196
+ "grad_norm": 0.17858931422233582,
197
+ "learning_rate": 2.664930555555556e-05,
198
+ "loss": 2.9046,
199
+ "step": 270
200
+ },
201
+ {
202
+ "epoch": 1.4583333333333333,
203
+ "grad_norm": 0.21472914516925812,
204
+ "learning_rate": 2.578125e-05,
205
+ "loss": 2.8826,
206
+ "step": 280
207
+ },
208
+ {
209
+ "epoch": 1.5104166666666665,
210
+ "grad_norm": 0.18860450387001038,
211
+ "learning_rate": 2.4913194444444446e-05,
212
+ "loss": 2.8265,
213
+ "step": 290
214
+ },
215
+ {
216
+ "epoch": 1.5625,
217
+ "grad_norm": 0.17438200116157532,
218
+ "learning_rate": 2.404513888888889e-05,
219
+ "loss": 2.7819,
220
+ "step": 300
221
+ },
222
+ {
223
+ "epoch": 1.6145833333333335,
224
+ "grad_norm": 0.23125068843364716,
225
+ "learning_rate": 2.3177083333333335e-05,
226
+ "loss": 2.8364,
227
+ "step": 310
228
+ },
229
+ {
230
+ "epoch": 1.6666666666666665,
231
+ "grad_norm": 0.18903665244579315,
232
+ "learning_rate": 2.2309027777777782e-05,
233
+ "loss": 2.8423,
234
+ "step": 320
235
+ },
236
+ {
237
+ "epoch": 1.71875,
238
+ "grad_norm": 0.22534161806106567,
239
+ "learning_rate": 2.144097222222222e-05,
240
+ "loss": 2.8666,
241
+ "step": 330
242
+ },
243
+ {
244
+ "epoch": 1.7708333333333335,
245
+ "grad_norm": 0.240483358502388,
246
+ "learning_rate": 2.0572916666666668e-05,
247
+ "loss": 2.8409,
248
+ "step": 340
249
+ },
250
+ {
251
+ "epoch": 1.8229166666666665,
252
+ "grad_norm": 0.23081159591674805,
253
+ "learning_rate": 1.970486111111111e-05,
254
+ "loss": 2.8257,
255
+ "step": 350
256
+ },
257
+ {
258
+ "epoch": 1.875,
259
+ "grad_norm": 0.20717695355415344,
260
+ "learning_rate": 1.8836805555555557e-05,
261
+ "loss": 2.9202,
262
+ "step": 360
263
+ },
264
+ {
265
+ "epoch": 1.9270833333333335,
266
+ "grad_norm": 0.21534092724323273,
267
+ "learning_rate": 1.796875e-05,
268
+ "loss": 2.7127,
269
+ "step": 370
270
+ },
271
+ {
272
+ "epoch": 1.9791666666666665,
273
+ "grad_norm": 0.24049048125743866,
274
+ "learning_rate": 1.7100694444444446e-05,
275
+ "loss": 2.8734,
276
+ "step": 380
277
+ },
278
+ {
279
+ "epoch": 2.03125,
280
+ "grad_norm": 0.21739767491817474,
281
+ "learning_rate": 1.623263888888889e-05,
282
+ "loss": 2.9164,
283
+ "step": 390
284
+ },
285
+ {
286
+ "epoch": 2.0833333333333335,
287
+ "grad_norm": 0.22166197001934052,
288
+ "learning_rate": 1.5364583333333335e-05,
289
+ "loss": 2.8199,
290
+ "step": 400
291
+ },
292
+ {
293
+ "epoch": 2.1354166666666665,
294
+ "grad_norm": 0.2430618405342102,
295
+ "learning_rate": 1.449652777777778e-05,
296
+ "loss": 2.7019,
297
+ "step": 410
298
+ },
299
+ {
300
+ "epoch": 2.1875,
301
+ "grad_norm": 0.21694371104240417,
302
+ "learning_rate": 1.3628472222222221e-05,
303
+ "loss": 2.6877,
304
+ "step": 420
305
+ },
306
+ {
307
+ "epoch": 2.2395833333333335,
308
+ "grad_norm": 0.24485336244106293,
309
+ "learning_rate": 1.2760416666666666e-05,
310
+ "loss": 2.8081,
311
+ "step": 430
312
+ },
313
+ {
314
+ "epoch": 2.2916666666666665,
315
+ "grad_norm": 0.23572859168052673,
316
+ "learning_rate": 1.189236111111111e-05,
317
+ "loss": 2.7529,
318
+ "step": 440
319
+ },
320
+ {
321
+ "epoch": 2.34375,
322
+ "grad_norm": 0.22991545498371124,
323
+ "learning_rate": 1.1024305555555557e-05,
324
+ "loss": 2.7204,
325
+ "step": 450
326
+ },
327
+ {
328
+ "epoch": 2.3958333333333335,
329
+ "grad_norm": 0.23390544950962067,
330
+ "learning_rate": 1.0156250000000001e-05,
331
+ "loss": 2.7938,
332
+ "step": 460
333
+ },
334
+ {
335
+ "epoch": 2.4479166666666665,
336
+ "grad_norm": 0.22788524627685547,
337
+ "learning_rate": 9.288194444444446e-06,
338
+ "loss": 2.7033,
339
+ "step": 470
340
+ },
341
+ {
342
+ "epoch": 2.5,
343
+ "grad_norm": 0.2522662281990051,
344
+ "learning_rate": 8.42013888888889e-06,
345
+ "loss": 2.7663,
346
+ "step": 480
347
+ },
348
+ {
349
+ "epoch": 2.5520833333333335,
350
+ "grad_norm": 0.23354235291481018,
351
+ "learning_rate": 7.552083333333333e-06,
352
+ "loss": 2.8088,
353
+ "step": 490
354
+ },
355
+ {
356
+ "epoch": 2.6041666666666665,
357
+ "grad_norm": 0.2031668871641159,
358
+ "learning_rate": 6.684027777777778e-06,
359
+ "loss": 2.7875,
360
+ "step": 500
361
+ },
362
+ {
363
+ "epoch": 2.65625,
364
+ "grad_norm": 0.2767188847064972,
365
+ "learning_rate": 5.815972222222223e-06,
366
+ "loss": 2.7891,
367
+ "step": 510
368
+ },
369
+ {
370
+ "epoch": 2.7083333333333335,
371
+ "grad_norm": 0.22986236214637756,
372
+ "learning_rate": 4.947916666666666e-06,
373
+ "loss": 2.6149,
374
+ "step": 520
375
+ },
376
+ {
377
+ "epoch": 2.7604166666666665,
378
+ "grad_norm": 0.2347174435853958,
379
+ "learning_rate": 4.079861111111111e-06,
380
+ "loss": 2.746,
381
+ "step": 530
382
+ },
383
+ {
384
+ "epoch": 2.8125,
385
+ "grad_norm": 0.19579485058784485,
386
+ "learning_rate": 3.2118055555555556e-06,
387
+ "loss": 2.7266,
388
+ "step": 540
389
+ },
390
+ {
391
+ "epoch": 2.8645833333333335,
392
+ "grad_norm": 0.22842058539390564,
393
+ "learning_rate": 2.3437500000000002e-06,
394
+ "loss": 2.7419,
395
+ "step": 550
396
+ },
397
+ {
398
+ "epoch": 2.9166666666666665,
399
+ "grad_norm": 0.22618690133094788,
400
+ "learning_rate": 1.4756944444444444e-06,
401
+ "loss": 2.7463,
402
+ "step": 560
403
+ },
404
+ {
405
+ "epoch": 2.96875,
406
+ "grad_norm": 0.2533385753631592,
407
+ "learning_rate": 6.076388888888889e-07,
408
+ "loss": 2.7705,
409
+ "step": 570
410
+ }
411
+ ],
412
+ "logging_steps": 10,
413
+ "max_steps": 576,
414
+ "num_input_tokens_seen": 0,
415
+ "num_train_epochs": 3,
416
+ "save_steps": 500,
417
+ "stateful_callbacks": {
418
+ "TrainerControl": {
419
+ "args": {
420
+ "should_epoch_stop": false,
421
+ "should_evaluate": false,
422
+ "should_log": false,
423
+ "should_save": true,
424
+ "should_training_stop": true
425
+ },
426
+ "attributes": {}
427
+ }
428
+ },
429
+ "total_flos": 4465501824614400.0,
430
+ "train_batch_size": 4,
431
+ "trial_name": null,
432
+ "trial_params": null
433
+ }