Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
@@ -8,7 +8,7 @@ import gradio as gr
|
|
8 |
import torch
|
9 |
from utils import *
|
10 |
from presets import *
|
11 |
-
from transformers import Trainer, TrainingArguments
|
12 |
import numpy as np
|
13 |
import evaluate
|
14 |
import pandas as pd
|
@@ -188,6 +188,7 @@ lm_datasets = tokenized_datasets.map(
|
|
188 |
num_proc=4,
|
189 |
)
|
190 |
|
|
|
191 |
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
192 |
|
193 |
|
|
|
8 |
import torch
|
9 |
from utils import *
|
10 |
from presets import *
|
11 |
+
from transformers import Trainer, TrainingArguments, DataCollatorWithPadding
|
12 |
import numpy as np
|
13 |
import evaluate
|
14 |
import pandas as pd
|
|
|
188 |
num_proc=4,
|
189 |
)
|
190 |
|
191 |
+
# Batches von Daten zusammenfassen
|
192 |
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
193 |
|
194 |
|