Upload retriever to Hugging Face Hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+database.lmdb/data.mdb filter=lfs diff=lfs merge=lfs -text
+indexes/bm25/vocab.index.json filter=lfs diff=lfs merge=lfs -text
+indexes/contriever/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,101 @@
+---
+language: en
+library_name: FlexRAG
+tags:
+- FlexRAG
+- retrieval
+- search
+- lexical
+- RAG
+- IR
+---
+
+# FlexRAG Retriever
+
+This is a FlexRetriever created with the [`FlexRAG`](https://github.com/ictnlp/FlexRAG) library (version `0.3.0`).
+
+## Retriever Attributes
+The `enwiki_2018_atlas` retriever is a FlexRetriever that provides access to the English Wikipedia corpus from December 2018. It is designed for information retrieval tasks, allowing users to search and retrieve relevant documents based on their queries.
+The corpus of this retriever was created by the [Atlas](https://github.com/facebookresearch/atlas) project and the index was built using the [FlexRAG](https://github.com/ictnlp/FlexRAG) library.
+
+| Corpus Attribute | Value                                                           |
+| ---------------- | --------------------------------------------------------------- |
+| Language         | English                                                         |
+| Domain           | Wikipedia                                                       |
+| Saved Fields     | title, section, text                                            |
+| Size             | 30.4M (26.9M text, 2.7M infobox)                                |
+| Dump Date        | Dec 2018                                                        |
+| Provideer        | [Atlas](https://github.com/facebookresearch/atlas)              |
+| License          | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+
+| Index Attribute | Value                                                           |
+| --------------- | --------------------------------------------------------------- |
+| Index Name      | bm25                                                            |
+| Index Type      | Sparse                                                          |
+| Index Method    | Lucene                                                          |
+| Indexed Fields  | title, section, text (concat)                                   |
+| Preprocessing   | LengthFilter(min_char=10, max_char=4096)                        |
+| Provideer       | [FlexRAG](https://github.com/ictnlp/flexrag)                    |
+| License         | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+| Index Attribute | Value                                                           |
+| --------------- | --------------------------------------------------------------- |
+| Index Name      | contriever                                                      |
+| Index Type      | Dense                                                           |
+| Index Method    | IVFPQ                                                           |
+| Indexed Fields  | title, section, text (concat)                                   |
+| Query Encoder   | `facebook/contriever-msmarco`                                   |
+| Passage Encoder | `facebook/contriever-msmarco`                                   |
+| Preprocessing   | LengthFilter(min_char=10, max_char=4096)                        |
+| Provideer       | [FlexRAG](https://github.com/ictnlp/flexrag)                    |
+| License         | [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) |
+
+## Usage
+
+### Installation
+You can install the `FlexRAG` library with `pip`:
+
+```bash
+pip install flexrag faiss-cpu
+```
+
+### Loading the `FlexRAG` retriever
+
+You can use this retriever for information retrieval tasks. Here is an example:
+
+```python
+from flexrag.retriever import LocalRetriever
+
+
+# Load the retriever from the HuggingFace Hub
+retriever = LocalRetriever.load_from_hub("FlexRAG/enwiki_2018_atlas")
+
+
+# You can retrieve relevant documents now
+results = retriever.search("Who is Bruce Wayne?")
+```
+
+### Running the RAG demo with the retriever
+
+You can run the **GUI application** of the RAG assistant with this retriever. Here is an example:
+
+```bash
+python -m flexrag.entrypoints.run_interactive \
+    assistant_type=modular \
+    modular_config.used_fields=[title,text] \
+    modular_config.retriever_type="FlexRAG/enwiki_2018_atlas" \
+    modular_config.response_type=original \
+    modular_config.generator_type=openai \
+    modular_config.openai_config.model_name='gpt-4o-mini' \
+    modular_config.openai_config.api_key=$OPENAI_KEY \
+    modular_config.do_sample=False
+```
+
+## License
+As the corpus is based on the [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) license, the retriever is also licensed under the same license.
+
+
+FlexRAG Related Links:
+* 📚[Documentation](https://flexrag.readthedocs.io/en/latest/)
+* 💻[GitHub Repository](https://github.com/ictnlp/flexrag)

cls.id

@@ -0,0 +1 @@
+FlexRetriever

config.yaml

@@ -0,0 +1,37 @@
+log_interval: 100000
+top_k: 10
+batch_size: 4096
+query_preprocess_pipeline:
+  processor_type: []
+  length_filter_config:
+    max_tokens: null
+    min_tokens: null
+    max_chars: null
+    min_chars: null
+    max_bytes: null
+    min_bytes: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+  token_normalize_config:
+    lang: en
+    penn: true
+    norm_quote_commas: true
+    norm_numbers: true
+    pre_replace_unicode_punct: false
+    post_remove_control_chars: false
+    perl_parity: false
+  truncate_config:
+    max_chars: null
+    max_bytes: null
+    max_tokens: null
+    tokenizer_config:
+      tokenizer_type: moses
+      hf_tokenizer_path: null
+      tiktok_tokenizer_name: null
+      lang: null
+retriever_path: null
+indexes_merge_method: linear
+used_indexes: null

database.lmdb/data.mdb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c266cb282533ef23d437ddd6e370816863a44276a7c82b7d611dff34562efdc5
+size 28694089728

indexes/bm25/cls.id

@@ -0,0 +1 @@
+BM25Index

indexes/bm25/config.yaml

@@ -0,0 +1,10 @@
+log_interval: 10000
+batch_size: 512
+index_path: /data/zhangzhuocheng/Lab/Python/LLM/datasets/RAG/Corpus/enwiki_2018_atlas/flex/indexes/bm25
+method: lucene
+idf_method: null
+backend: auto
+k1: 1.5
+b: 0.75
+delta: 0.5
+lang: english

indexes/bm25/context_mapping.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9813df1610580866468946adab882bfeb9ef4b067217154ce93f0b5ff63810
+size 942169490

indexes/bm25/data.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa0a05322a20d4abc5866f1e7b7ec5b44b813a913b1b48f576a1ca8078889948
+size 6591838820

indexes/bm25/indices.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149c01e2f984f32106a0648651f517965d11c538c16e14a7f431e5dc98f44878
+size 6591838820

indexes/bm25/indptr.csc.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65855cea4e1ec6645c39ffa02fdd6c99d279bf82cc4351155c00fad49dded9ab
+size 41208084

indexes/bm25/multi_field_index_config.yaml

@@ -0,0 +1,5 @@
+indexed_fields:
+- title
+- section
+- text
+merge_method: concat

indexes/bm25/params.index.json

@@ -0,0 +1,12 @@
+{
+    "k1": 1.5,
+    "b": 0.75,
+    "delta": 0.5,
+    "method": "lucene",
+    "idf_method": "lucene",
+    "dtype": "float32",
+    "int_dtype": "int32",
+    "num_docs": 32023701,
+    "version": "0.2.11",
+    "backend": "numpy"
+}

indexes/bm25/vocab.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2430ed7b035d343553756556f1c0e404d3099c660d72fe70c9c799b7a108250
+size 208594633

indexes/contriever/cls.id

@@ -0,0 +1 @@
+FaissIndex

indexes/contriever/config.yaml

@@ -0,0 +1,62 @@
+log_interval: 100000
+batch_size: 2048
+index_path: null
+query_encoder_config:
+  encoder_type: hf
+  cohere_config: null
+  hf_config:
+    batch_size: 32
+    log_interval: 1000
+    model_path: facebook/contriever-msmarco
+    tokenizer_path: null
+    trust_remote_code: false
+    device_id:
+    - 0
+    load_dtype: auto
+    max_encode_length: 512
+    encode_method: mean
+    normalize: false
+    prompt: ''
+    task: ''
+  hf_clip_config: null
+  jina_config: null
+  ollama_config: null
+  openai_config: null
+  sentence_transformer_config: null
+passage_encoder_config:
+  encoder_type: hf
+  cohere_config: null
+  hf_config:
+    batch_size: 32
+    log_interval: 1000
+    model_path: facebook/contriever-msmarco
+    tokenizer_path: null
+    trust_remote_code: false
+    device_id:
+    - 0
+    - 1
+    - 2
+    - 3
+    load_dtype: auto
+    max_encode_length: 512
+    encode_method: mean
+    normalize: false
+    prompt: ''
+    task: ''
+  hf_clip_config: null
+  jina_config: null
+  ollama_config: null
+  openai_config: null
+  sentence_transformer_config: null
+distance_function: IP
+index_type: auto
+n_subquantizers: 8
+n_bits: 8
+n_list: 1000
+factory_str: null
+index_train_num: -1
+n_probe: 512
+device_id: []
+k_factor: 10
+polysemous_ht: 0
+efSearch: 100

indexes/contriever/context_mapping.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9813df1610580866468946adab882bfeb9ef4b067217154ce93f0b5ff63810
+size 942169490

indexes/contriever/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a3029e71c0d3d079099652650489cba2213e75320d4def527c19f00e8c26a0
+size 6429691256

indexes/contriever/multi_field_index_config.yaml

@@ -0,0 +1,5 @@
+indexed_fields:
+- title
+- section
+- text
+merge_method: concat