alexkueck commited on
Commit
c856a9e
·
1 Parent(s): 4f0d6ba

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -2
app.py CHANGED
@@ -89,11 +89,15 @@ base_model = "project-baize/baize-v2-7b" #load_8bit = True (in load_tokenizer_a
89
 
90
 
91
  # Load model directly
92
-
93
  #Tokenizer und Model laden
94
  tokenizer,model,device = load_tokenizer_and_model(base_model, True)
95
- tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
 
 
 
96
 
 
97
  #Datensets für Finetuning laden
98
  dataset_neu = daten_laden("alexkueck/tis")
99
  #dataset_neu = daten_laden("EleutherAI/pile")
 
89
 
90
 
91
  # Load model directly
92
+ #####################################################
93
  #Tokenizer und Model laden
94
  tokenizer,model,device = load_tokenizer_and_model(base_model, True)
95
+ #tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
96
+ #für Blaize....
97
+ tokenizer,model,device = load_tokenizer_and_model_Blaize(base_model, True)
98
+ tokenizer.pad_token_id = 0
99
 
100
+ ####################################################
101
  #Datensets für Finetuning laden
102
  dataset_neu = daten_laden("alexkueck/tis")
103
  #dataset_neu = daten_laden("EleutherAI/pile")