Spaces:
Sleeping
Sleeping
File size: 2,029 Bytes
d85b3ec |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
"""
Sep 2024
Ryan Tietjen
Contains helper functions to process user input for the demo
"""
import pandas as pd
def split_abstract(abstract):
results = []
lines = abstract.split("\n")
for i, line in enumerate(lines):
entry = {
"target": 0,
"text": line.lower(),
"line_number": i + 1,
"total_lines": len(lines)
}
results.append(entry)
return results
def split_abstract_original(abstract):
results = []
lines = abstract.split("\n")
for i, line in enumerate(lines):
entry = {
"target": 0,
"text": line,
"line_number": i + 1,
"total_lines": len(lines)
}
results.append(entry)
return results
def split_sentences_by_characters(corpus):
return [" ".join(sentence) for sentence in corpus]
def encode_labels(*datasets):
"""
Encode labels for multiple datasets using a unified label mapping.
Args:
*datasets: Arbitrary number of array-like structures containing labels.
Returns:
tuple: Encoded labels as numpy arrays for each dataset.
"""
# Collect all labels from all datasets into a single list
all_labels = pd.concat([pd.Series(data) for data in datasets])
# Get unique labels and sort them to ensure consistency
unique_labels = pd.unique(all_labels)
unique_labels.sort()
# Create mapping from labels to integers
label_to_index = {label: idx for idx, label in enumerate(unique_labels)}
# Function to encode a single dataset
def encode_single_dataset(dataset, mapping):
return pd.Series(dataset).map(mapping).to_numpy()
# Encode all datasets using the mapping
encoded_datasets = tuple(encode_single_dataset(dataset, label_to_index) for dataset in datasets)
# Return only the encoded datasets
return encoded_datasets |