Spaces:
Configuration error
Configuration error
Update cross_encoder_reranking_train.py
Browse files
cross_encoder_reranking_train.py
CHANGED
@@ -1,19 +1,19 @@
|
|
1 |
import os
|
2 |
import json
|
|
|
3 |
import argparse
|
4 |
import numpy as np
|
5 |
-
import torch
|
6 |
-
import torch.nn.functional as F
|
7 |
from tqdm import tqdm
|
8 |
from torch import Tensor
|
9 |
-
|
10 |
from transformers import AutoTokenizer, AutoModel
|
11 |
from sklearn.cluster import AgglomerativeClustering
|
|
|
12 |
from sklearn.metrics.pairwise import cosine_similarity
|
13 |
|
14 |
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
15 |
# Load embedder once
|
16 |
-
# embedder = SentenceTransformer("all-MiniLM-
|
17 |
embedder = SentenceTransformer("AI-Growth-Lab/PatentSBERTa").to(device)
|
18 |
|
19 |
|
|
|
1 |
import os
|
2 |
import json
|
3 |
+
import torch
|
4 |
import argparse
|
5 |
import numpy as np
|
|
|
|
|
6 |
from tqdm import tqdm
|
7 |
from torch import Tensor
|
8 |
+
import torch.nn.functional as F
|
9 |
from transformers import AutoTokenizer, AutoModel
|
10 |
from sklearn.cluster import AgglomerativeClustering
|
11 |
+
from sentence_transformers import SentenceTransformer
|
12 |
from sklearn.metrics.pairwise import cosine_similarity
|
13 |
|
14 |
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
15 |
# Load embedder once
|
16 |
+
# embedder = SentenceTransformer("all-MiniLM-L12-v2").to(device)
|
17 |
embedder = SentenceTransformer("AI-Growth-Lab/PatentSBERTa").to(device)
|
18 |
|
19 |
|