Add H-optimus-0-SPECTRA encoder

README.md

@@ -65,6 +65,7 @@ An API and command line interface (CLI) are provided to allow users to download
 * Virchow2: https://huggingface.co/paige-ai/Virchow2
 * H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0
 * H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1
+* H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA
 * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA
 * Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA
 * GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm

src/thunder/benchmark.py

@@ -21,7 +21,7 @@ def benchmark(
 
     where options are:
         - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds*
-        - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14*
+        - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14*
         - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm*
         - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading*
 

src/thunder/config/pretrained_model/hoptimus0_spectra.yaml

@@ -0,0 +1,8 @@
+model_name: hoptimus0_spectra
+type: timm
+vlm: false
+emb_dim: 1536
+ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/hoptimus0spectra/model.safetensors
+hf_tag: hf-hub:medarc/H-optimus-0-SPECTRA
+
+timm_kwargs: {}

src/thunder/models/download.py

@@ -30,6 +30,10 @@
         "paige-ai/Virchow2",
         "pytorch_model.bin",
     ),  # Virchow2 (https://huggingface.co/paige-ai/Virchow2)
+    "hoptimus0_spectra": (
+        "medarc/H-optimus-0-SPECTRA",
+        "model.safetensors",
+    ),  # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA)
     "hoptimus0": (
         "bioptimus/H-optimus-0",
         "pytorch_model.bin",
@@ -167,6 +171,7 @@ def download_models(models: Union[List[str], str]) -> None:
         * virchow_spectra
         * virchow2
         * hoptimus0
+        * hoptimus0_spectra
         * h0mini
         * hoptimus1
         * genbio-pathfm
@@ -272,7 +277,7 @@ def download_model(model: str) -> None:
                 ]
                 if local_dir_tag != "plip":
                     extra_files.extend(["merges.txt", "vocab.json"])
-            elif local_dir_tag in ["midnight", "midnight12kspectra", "virchowspectra"]:
+            elif local_dir_tag in ["midnight", "hoptimus0spectra", "midnight12kspectra", "virchowspectra"]:
                 extra_files = ["config.json"]
             elif local_dir_tag == "titan":
                 extra_files = [

src/thunder/models/pretrained_models.py

@@ -377,7 +377,11 @@ def extract_embedding(src, pretrained_model, task_type="linear_probing"):
 
                     if model_cfg.model_name == "uni2h":
                         nb_reg = 8
-                    elif model_cfg.model_name in ["hoptimus0", "hoptimus1"]:
+                    elif model_cfg.model_name in [
+                        "hoptimus0",
+                        "hoptimus1",
+                        "hoptimus0_spectra",
+                    ]:
                         nb_reg = 4
                     else:
                         nb_reg = 0
@@ -399,6 +403,7 @@ def get_model_from_name(model_name: str, device: str):
         * virchow_spectra
         * virchow2
         * hoptimus0
+        * hoptimus0_spectra
         * h0mini
         * hoptimus1
         * genbio-pathfm

src/thunder/utils/constants.py

@@ -7,6 +7,7 @@ class ModelConstants(Enum):
         "hiboub",
         "hiboul",
         "hoptimus0",
+        "hoptimus0_spectra",
         "h0mini",
         "hoptimus1",
         "genbio-pathfm",