docs: Add MTEB(code) dataset (#1237)

* docs: Add MTEB(code) dataset * Fix linting
embeddings-benchmark · Sep 24, 2024 · f808863 · f808863
1 parent dddf5c2
commit f808863
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 1 deletion.
diff --git a/docs/mmteb/points.md b/docs/mmteb/points.md
@@ -119,4 +119,4 @@ Please also add your first name and last name are as you want them to appear in
 | mrshu | Marek | Suppa | [email protected] | ~Marek_Suppa1 | Comenius University in Bratislava && Cisco Systems |
 | swj0419 | Weijia | Shi | [email protected] | ~Weijia_Shi1 | University of Washington |
 | xiamengzhou | Mengzhou | Xia | [email protected] | ~Mengzhou_Xia1 | Princeton University |
-| john-b-yang | John | Yang | | ~John_Yang3 | Princeton University |
+| john-b-yang | John | Yang | [email protected] | ~John_Yang3 | Stanford University |
diff --git a/docs/mmteb/points/1237.jsonl b/docs/mmteb/points/1237.jsonl
@@ -0,0 +1 @@
+{"GitHub": "john-b-yang", "Paper writing": 20}
diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py
@@ -505,3 +505,42 @@ def __getitem__(self, index):
     year={2024}
 }""",
 )
+
+MTEB_code = Benchmark(
+    name="MTEB(code)",
+    tasks=get_tasks(
+        tasks=[
+            # Retrieval
+            "AppsRetrieval",
+            "CodeEditSearchRetrieval",
+            "CodeFeedbackMT",
+            "CodeFeedbackST",
+            "CodeSearchNetCCRetrieval",
+            "CodeSearchNetRetrieval",
+            "CodeTransOceanContest",
+            "CodeTransOceanDL",
+            "CosQA",
+            "COIRCodeSearchNetRetrieval",
+            "StackOverflowQA",
+            "SyntheticText2SQL",
+        ],
+        languages=[
+            "c",
+            "c++",
+            "go",
+            "java",
+            "javascript",
+            "php",
+            "python",
+            "ruby",
+            "rust",
+            "scala",
+            "shell",
+            "swift",
+            "typescript",
+        ],
+    ),
+    description="Main code benchmarks from MTEB",
+    reference=None,
+    citation=None,
+)