build dataset on machine

Dockerfile

@@ -8,11 +8,12 @@ ENV HOME=/code
 RUN apt install curl
 RUN pip install pip -U
 
-RUN pip install renumics-spotlight pyarrow
+RUN pip install renumics-spotlight==1.3.0rc8 pyarrow
 
-# RUN pip install datasets
+RUN pip install datasets
 
 COPY . .
 RUN mkdir -p /code/.cache
 RUN chmod -R 777 /code
+RUN python prepare.py
 CMD ["python", "run.py"]

cifar100-enriched-cv.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a00c5f027c054d5a697540d1396d07f70cd651ff59b6b862f81498e164279351
-size 37007568

prepare.py

@@ -0,0 +1,44 @@
+import pickle
+import datasets
+import os
+from cleanvision.imagelab import Imagelab
+import pandas as pd
+
+
+
+
+def cv_issues_cleanvision(df, image_name='image'):
+
+    image_paths = df['image'].to_list()
+    imagelab = Imagelab(filepaths=image_paths)
+    imagelab.find_issues()
+
+    df_cv=imagelab.issues.reset_index()
+
+    return df_cv
+    
+
+if __name__ == "__main__":
+    cache_file = "dataset_cache.parquet"
+    if os.path.exists(cache_file):
+        # Load dataset from cache
+        df = pd.read_parquet('cifar100-enriched-cv.parquet')
+        print("Dataset loaded from cache.")
+    else:
+        # Load dataset using datasets.load_dataset()
+        dataset = datasets.load_dataset("renumics/cifar100-enriched", split="test")
+        print("Dataset loaded using datasets.load_dataset().")
+        
+        df = dataset.to_pandas()
+        
+        df=df.reset_index(drop=True)
+        
+        df_cv=cv_issues_cleanvision(df)
+        df = pd.concat([df, df_cv], axis=1)
+
+        # Save dataset to cache
+        #save df as parquet
+        df.to_parquet(cache_file)
+
+        print("Dataset saved to cache.")
+

run.py

@@ -4,22 +4,19 @@ import pickle
 from renumics import spotlight
 
 if __name__ == "__main__":
-    cache_file = "dataset_cache.pkl"
+    cache_file = "dataset_cache.parquet"
     issue_cache_file="sliceline.pkl"
     
     import pandas as pd
 
     #load dataframe from parquet
-    df = pd.read_parquet('cifar100-enriched-cv.parquet')    
+    df = pd.read_parquet(cache_file)    
     
-    #with open(issue_cache_file, "rb") as issue_file:
-    #     issues = pickle.load(issue_file)
  
     df_show = df.drop(columns=['embedding', 'probabilities'])
     
     while True:
-        #view = spotlight.show(df_show, port=7860, host="0.0.0.0", issues=issues, layout="sliceline-layout.json",
-        #            dtype={"image": spotlight.Image, "embedding_reduced": spotlight.Embedding}, allow_filebrowsing=False)
-        view = spotlight.show(df_show, port=7860, host="0.0.0.0", 
+        view = spotlight.show(df_show, port=7860, host="0.0.0.0", issues=issues, layout="sliceline-layout.json",
                     dtype={"image": spotlight.Image, "embedding_reduced": spotlight.Embedding}, allow_filebrowsing=False)
+        
         view.close()