Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
@@ -89,11 +89,15 @@ base_model = "project-baize/baize-v2-7b" #load_8bit = True (in load_tokenizer_a
|
|
89 |
|
90 |
|
91 |
# Load model directly
|
92 |
-
|
93 |
#Tokenizer und Model laden
|
94 |
tokenizer,model,device = load_tokenizer_and_model(base_model, True)
|
95 |
-
tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
|
|
|
|
|
|
|
96 |
|
|
|
97 |
#Datensets für Finetuning laden
|
98 |
dataset_neu = daten_laden("alexkueck/tis")
|
99 |
#dataset_neu = daten_laden("EleutherAI/pile")
|
|
|
89 |
|
90 |
|
91 |
# Load model directly
|
92 |
+
#####################################################
|
93 |
#Tokenizer und Model laden
|
94 |
tokenizer,model,device = load_tokenizer_and_model(base_model, True)
|
95 |
+
#tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
|
96 |
+
#für Blaize....
|
97 |
+
tokenizer,model,device = load_tokenizer_and_model_Blaize(base_model, True)
|
98 |
+
tokenizer.pad_token_id = 0
|
99 |
|
100 |
+
####################################################
|
101 |
#Datensets für Finetuning laden
|
102 |
dataset_neu = daten_laden("alexkueck/tis")
|
103 |
#dataset_neu = daten_laden("EleutherAI/pile")
|