Merge branch 'main' into simplify-models

embeddings-benchmark · Jul 24, 2024 · 886c796 · 886c796
2 parents 027110a + 2e6dd12
commit 886c796
Show file tree

Hide file tree

Showing 28 changed files with 787 additions and 403 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -8,7 +8,8 @@ on:
   pull_request:
 
 jobs:
-  create-table:
+  create-table-on-pr:
+    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -23,23 +24,40 @@ jobs:
           make install
 
       - name: Create table
-        run: python docs/create_tasks_table.py
+        run: |
+          make build-docs
+
+  create-table-and-push:
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.RELEASE }}
 
-      - name: Configure Git (for push only)
-        if: github.event_name == 'push'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
         run: |
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git config --global user.name "github-actions[bot]"
+          make install
 
+      - name: Create table
+        run: |
+          make build-docs
+      
       - name: Push table
-        if: github.event_name == 'push'
-        env:
-          GITHUB_TOKEN: ${{ secrets.RELEASE }}
         run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
           # Check if changes exist
           if git diff --quiet; then
             echo "No changes detected"
           else
             git add docs/tasks.md
             git commit -m "Update tasks table"
-            git push
+            git push
+          fi
+          
diff --git a/.gitignore b/.gitignore
@@ -141,3 +141,6 @@ tmp.py
 # sandbox
 sb.ipynb
 tests/create_meta/model_card.md
+
+# removed results from mteb repo they are now available at: https://github.com/embeddings-benchmark/results
+results/
diff --git a/Makefile b/Makefile
@@ -30,3 +30,9 @@ pr:
 	@echo "--- 🚀 Running requirements for a PR ---"
 	make lint
 	make test
+
+
+build-docs:
+	@echo "--- 📚 Building documentation ---"
+	# since we do not have a documentation site, this just build tables for the .md files
+	python docs/create_tasks_table.py
diff --git a/docs/mmteb/points/1072.jsonl b/docs/mmteb/points/1072.jsonl
@@ -0,0 +1 @@
+{"GitHub": "Samoed", "Bug fixes": 2}
diff --git a/docs/mmteb/points_table.md b/docs/mmteb/points_table.md
@@ -57,9 +57,9 @@ _Note_: this table is **autogenerated** and should not be edited. It is intended
 | sted97            |            16 |           0 |           0 |          0 |                     0 |              0 |                0 |               0 |      16 |
 | PranjalChitale    |            16 |           0 |           0 |          0 |                     0 |              0 |                0 |               0 |      16 |
 | Sakshamrzt        |            12 |           4 |           0 |          0 |                     0 |              0 |                0 |               0 |      16 |
+| Samoed            |             0 |           0 |           6 |          0 |                     0 |              0 |                9 |               0 |      15 |
 | taidnguyen        |            14 |           0 |           0 |          0 |                     0 |              0 |                0 |               0 |      14 |
 | artemsnegirev     |            12 |           0 |           0 |          0 |                     2 |              0 |                0 |               0 |      14 |
-| Samoed            |             0 |           0 |           4 |          0 |                     0 |              0 |                9 |               0 |      13 |
 | mariyahendriksen  |             0 |           0 |           0 |          0 |                     0 |              0 |                0 |              12 |      12 |
 | xhluca            |             6 |           2 |           4 |          0 |                     0 |              0 |                0 |               0 |      12 |
 | anpalmak2003      |             9 |           0 |           0 |          0 |                     3 |              0 |                0 |               0 |      12 |

diff --git a/docs/tasks.md b/docs/tasks.md
@@ -52,6 +52,7 @@ The following tables give you an overview of the tasks in MTEB.
 | [BlurbsClusteringS2S.v2](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/germeval-2019-hmc.html) (Steffen Remus, 2019) | ['deu'] | Clustering | s2s | [Fiction, Written] | {'test': 2048} | {'test': 23.02} |
 | [BornholmBitextMining](https://aclanthology.org/W19-6138/) | ['dan'] | BitextMining | s2s | [Web, Social, Fiction, Written] | {'test': 500} | {'test': 89.7} |
 | [BrazilianToxicTweetsClassification](https://paperswithcode.com/dataset/told-br) (Joao Augusto Leite and                         Diego F. Silva and                         Kalina Bontcheva and                         Carolina Scarton, 2020) | ['por'] | MultilabelClassification | s2s | [Constructed, Written] | {'test': 2048} | {'test': 85.05} |
+| [BrightRetrieval](https://huggingface.co/datasets/xlangai/BRIGHT) (Su et al., 2024) | ['eng'] | Retrieval | s2p | [Non-fiction] | {'standard': 1334914, 'long': 7048} | {'standard': 800.3994729248476, 'long': 46527.35839954597} |
 | [BulgarianStoreReviewSentimentClassfication](https://doi.org/10.7910/DVN/TXIK9P) (Georgieva-Trifonova et al., 2018) | ['bul'] | Classification | s2s | [Reviews, Written] | {'test': 182} | {'test': 316.7} |
 | [CBD](http://2019.poleval.pl/files/poleval2019.pdf) | ['pol'] | Classification | s2s |  | {'test': 1000} | {'test': 93.2} |
 | [CDSC-E](https://aclanthology.org/P17-1073.pdf) | ['pol'] | PairClassification | s2s |  | None | None |
@@ -850,7 +851,7 @@ The following tables give you an overview of the tasks in MTEB.
 | ell | 3 | 6 | 1 | 0 | 1 | 2 | 0 | 3 | 0 | 0 | 0 |
 | emi | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
 | emp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| eng | 16 | 143 | 16 | 3 | 1 | 8 | 7 | 72 | 13 | 2 | 1 |
+| eng | 16 | 143 | 16 | 3 | 1 | 8 | 7 | 73 | 13 | 2 | 1 |
 | enq | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
 | epo | 3 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
 | eri | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
@@ -1622,7 +1623,7 @@ The following tables give you an overview of the tasks in MTEB.
 | zty | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
 | zul | 2 | 3 | 1 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 |
 | zyp | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
-| Total | 1394 | 794 | 304 | 3 | 28 | 67 | 47 | 371 | 85 | 2 | 2 |
+| Total | 1394 | 794 | 304 | 3 | 28 | 67 | 47 | 372 | 85 | 2 | 2 |
 <!-- TASK LANG TABLE END -->
 
 </details>
diff --git a/missing_tasks.txt b/missing_tasks.txt