Update app.py
Browse files
app.py
CHANGED
|
@@ -1,20 +1,5 @@
|
|
|
|
|
| 1 |
import os
|
| 2 |
-
|
| 3 |
-
os.system("python -m pip install transformers")
|
| 4 |
-
os.system("python -m pip install spacy==3.5.4")
|
| 5 |
-
os.system("python -m pip install spacy-alignments==0.9.1")
|
| 6 |
-
os.system("python -m pip install spacy-legacy==3.0.12")
|
| 7 |
-
os.system("python -m pip install spacy-loggers==1.0.3")
|
| 8 |
-
os.system("python -m pip install torch")
|
| 9 |
-
os.system("python -m pip install seaborn==0.11.2")
|
| 10 |
-
os.system("python -m pip install gradio==3.16.1")
|
| 11 |
-
os.system("python -m pip install typer==0.4.1")
|
| 12 |
-
os.system("python -m pip install pydantic==1.9.2")
|
| 13 |
-
os.system("python -m pip install matplotlib==3.4.3")
|
| 14 |
-
os.system("python -m pip install Flask")
|
| 15 |
-
os.system("python -m pip install sty")
|
| 16 |
-
|
| 17 |
-
from flask import Flask, render_template, request, Response, send_file, jsonify
|
| 18 |
import hashlib
|
| 19 |
from pipeline import *
|
| 20 |
import csv
|
|
@@ -29,7 +14,7 @@ if not os.path.isdir(app.config['UPLOAD_FOLDER']):
|
|
| 29 |
os.makedirs(app.config['UPLOAD_FOLDER'])
|
| 30 |
|
| 31 |
# Allowable file extensions for uploading
|
| 32 |
-
ALLOWED_EXTENSIONS = {'txt'}
|
| 33 |
|
| 34 |
# Check if a file has an allowable extension
|
| 35 |
def allowed_file(filename):
|
|
@@ -38,13 +23,31 @@ def allowed_file(filename):
|
|
| 38 |
|
| 39 |
# Reverse the lines in the file and return a list of dictionaries containing the original input and the processed output
|
| 40 |
def process_file(file_path):
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
result = [{'input': line.strip(), 'output': result} for line, result in zip(lines, results)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
return result
|
| 49 |
|
| 50 |
# Home page route that allows users to upload files
|
|
@@ -69,16 +72,16 @@ def upload_file():
|
|
| 69 |
|
| 70 |
# Generate a unique hash code for the file name
|
| 71 |
hash_code = hashlib.md5(file.read()).hexdigest()
|
|
|
|
| 72 |
filename = f"{hash_code}.txt"
|
|
|
|
|
|
|
| 73 |
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
|
| 74 |
|
| 75 |
# Save the uploaded file
|
| 76 |
file.seek(0)
|
| 77 |
file.save(file_path)
|
| 78 |
|
| 79 |
-
# Create a temporary HTML page to inform the user that the file is being processed
|
| 80 |
-
tmp_html = "<html><body><h1>Please wait while your file is being processed...</h1></body></html>"
|
| 81 |
-
|
| 82 |
# Process the uploaded file and return the result as a JSON line file
|
| 83 |
result = process_file(file_path)
|
| 84 |
result_file_path = os.path.join(app.config['UPLOAD_FOLDER'], 'result.csv')
|
|
|
|
| 1 |
+
from flask import Flask, render_template, request, send_file, after_this_request
|
| 2 |
import os
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import hashlib
|
| 4 |
from pipeline import *
|
| 5 |
import csv
|
|
|
|
| 14 |
os.makedirs(app.config['UPLOAD_FOLDER'])
|
| 15 |
|
| 16 |
# Allowable file extensions for uploading
|
| 17 |
+
ALLOWED_EXTENSIONS = {'txt','csv'}
|
| 18 |
|
| 19 |
# Check if a file has an allowable extension
|
| 20 |
def allowed_file(filename):
|
|
|
|
| 23 |
|
| 24 |
# Reverse the lines in the file and return a list of dictionaries containing the original input and the processed output
|
| 25 |
def process_file(file_path):
|
| 26 |
+
file_handle = open(file_path,'r',encoding='utf-8-sig',errors='ignore')
|
| 27 |
+
|
| 28 |
+
with open(file_path, 'r', encoding='utf-8-sig',errors='ignore') as f:
|
| 29 |
+
if file_path.endswith(".txt"):
|
| 30 |
+
lines = f.readlines()
|
| 31 |
+
results = []
|
| 32 |
+
for line in lines:
|
| 33 |
+
if line.strip() != "":
|
| 34 |
+
result = run_pipeline(line.strip())
|
| 35 |
+
results.append(result)
|
| 36 |
+
elif file_path.endswith(".csv"):
|
| 37 |
+
reader = csv.reader(f)
|
| 38 |
+
for line in reader:
|
| 39 |
+
if line[0].strip() != "":
|
| 40 |
+
result = run_pipeline(line[0].strip())
|
| 41 |
+
results.append(result)
|
| 42 |
result = [{'input': line.strip(), 'output': result} for line, result in zip(lines, results)]
|
| 43 |
+
@after_this_request
|
| 44 |
+
def remove_file(response):
|
| 45 |
+
try:
|
| 46 |
+
os.remove(file_path)
|
| 47 |
+
file_handle.close()
|
| 48 |
+
except Exception as error:
|
| 49 |
+
app.logger.error("Error removing or closing downloaded file handle", error)
|
| 50 |
+
return response
|
| 51 |
return result
|
| 52 |
|
| 53 |
# Home page route that allows users to upload files
|
|
|
|
| 72 |
|
| 73 |
# Generate a unique hash code for the file name
|
| 74 |
hash_code = hashlib.md5(file.read()).hexdigest()
|
| 75 |
+
# if ".txt" in filename:
|
| 76 |
filename = f"{hash_code}.txt"
|
| 77 |
+
# elif ".csv" in filename:
|
| 78 |
+
# filename = f"{hash_code}.csv"
|
| 79 |
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
|
| 80 |
|
| 81 |
# Save the uploaded file
|
| 82 |
file.seek(0)
|
| 83 |
file.save(file_path)
|
| 84 |
|
|
|
|
|
|
|
|
|
|
| 85 |
# Process the uploaded file and return the result as a JSON line file
|
| 86 |
result = process_file(file_path)
|
| 87 |
result_file_path = os.path.join(app.config['UPLOAD_FOLDER'], 'result.csv')
|