Update human_text_detect.py
Browse files- human_text_detect.py +4 -4
human_text_detect.py
CHANGED
@@ -15,7 +15,6 @@ import os
|
|
15 |
|
16 |
logging.basicConfig(level=logging.INFO)
|
17 |
|
18 |
-
|
19 |
def read_all_csv_files(pattern):
|
20 |
df = pd.DataFrame()
|
21 |
print(pattern)
|
@@ -23,7 +22,6 @@ def read_all_csv_files(pattern):
|
|
23 |
df = pd.concat([df, pd.read_csv(f)])
|
24 |
return df
|
25 |
|
26 |
-
|
27 |
def get_survival_function(df, G=101):
|
28 |
"""
|
29 |
Returns a survival function for every sentence length in tokens.
|
@@ -44,7 +42,6 @@ def get_survival_function(df, G=101):
|
|
44 |
xx1 = df1[value_name]
|
45 |
return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
|
46 |
|
47 |
-
|
48 |
def mark_edits_remove_tags(chunks, tag="edit"):
|
49 |
text_chunks = chunks['text']
|
50 |
edits = []
|
@@ -96,7 +93,10 @@ def detect_human_text(model_name, topic, text):
|
|
96 |
min_tokens_per_sentence = 10
|
97 |
max_tokens_per_sentence = 100
|
98 |
|
99 |
-
cache_dir = f"
|
|
|
|
|
|
|
100 |
|
101 |
# Init model
|
102 |
print('Init model')
|
|
|
15 |
|
16 |
logging.basicConfig(level=logging.INFO)
|
17 |
|
|
|
18 |
def read_all_csv_files(pattern):
|
19 |
df = pd.DataFrame()
|
20 |
print(pattern)
|
|
|
22 |
df = pd.concat([df, pd.read_csv(f)])
|
23 |
return df
|
24 |
|
|
|
25 |
def get_survival_function(df, G=101):
|
26 |
"""
|
27 |
Returns a survival function for every sentence length in tokens.
|
|
|
42 |
xx1 = df1[value_name]
|
43 |
return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
|
44 |
|
|
|
45 |
def mark_edits_remove_tags(chunks, tag="edit"):
|
46 |
text_chunks = chunks['text']
|
47 |
edits = []
|
|
|
93 |
min_tokens_per_sentence = 10
|
94 |
max_tokens_per_sentence = 100
|
95 |
|
96 |
+
cache_dir = f"/cacheHuggingface/{model_name}"
|
97 |
+
# Use a writable directory inside the Hugging Face Space
|
98 |
+
os.makedirs("/cacheHuggingface/PHI2", exist_ok=True)
|
99 |
+
os.makedirs("/cacheHuggingface/GPT2XL", exist_ok=True)
|
100 |
|
101 |
# Init model
|
102 |
print('Init model')
|