Update app.py

app.py

@@ -2363,11 +2363,40 @@ def similarity_matching(sprites_data: str, project_folder: str) -> str:
     logger.info("🔍 Running similarity matching…")
     os.makedirs(project_folder, exist_ok=True)
 
+    # # ----------------------------------------
+    # # CHANGED: define normalized base-paths so startswith() checks work
+    # backdrop_base_path = os.path.normpath(str(BACKDROP_DIR))
+    # sprite_base_path   = os.path.normpath(str(SPRITE_DIR))
+    # code_blocks_path = os.path.normpath(str(CODE_BLOCKS_DIR))
+    # # ----------------------------------------
+
+    # project_json_path = os.path.join(project_folder, "project.json")
+
+    # # ==============================
+    # #    READ SPRITE METADATA
+    # # ==============================
+    # # with open(input_json_path, 'r') as f:
+    # #     sprites_data = json.load(f)
+
+    # sprite_ids, sprite_base64 = [], []
+    # for sid, sprite in sprites_data.items():
+    #     sprite_ids.append(sid)
+    #     # texts.append("This is " + sprite.get("description", sprite.get("name", "")))
+    #     sprite_base64.append(sprite["base64"])
+
+    # sprite_images_bytes = []
+    # for b64 in sprite_base64:
+    #     img = Image.open(BytesIO(base64.b64decode(b64.split(",")[-1]))).convert("RGB")
+    #     buffer = BytesIO()
+    #     img.save(buffer, format="PNG")
+    #     buffer.seek(0)
+    #     sprite_images_bytes.append(buffer)
+
     # ----------------------------------------
-    # CHANGED: define normalized base-paths so startswith() checks work
+    # normalized base-paths
     backdrop_base_path = os.path.normpath(str(BACKDROP_DIR))
     sprite_base_path   = os.path.normpath(str(SPRITE_DIR))
-    code_blocks_path = os.path.normpath(str(CODE_BLOCKS_DIR))
+    code_blocks_path   = os.path.normpath(str(CODE_BLOCKS_DIR))
     # ----------------------------------------
 
     project_json_path = os.path.join(project_folder, "project.json")
@@ -2375,23 +2404,26 @@ def similarity_matching(sprites_data: str, project_folder: str) -> str:
     # ==============================
     #    READ SPRITE METADATA
     # ==============================
-    # with open(input_json_path, 'r') as f:
-    #     sprites_data = json.load(f)
-
     sprite_ids, sprite_base64 = [], []
     for sid, sprite in sprites_data.items():
         sprite_ids.append(sid)
-        # texts.append("This is " + sprite.get("description", sprite.get("name", "")))
         sprite_base64.append(sprite["base64"])
 
+    # === decode base64 into BytesIO list (you already did similar) ===
     sprite_images_bytes = []
     for b64 in sprite_base64:
-        img = Image.open(BytesIO(base64.b64decode(b64.split(",")[-1]))).convert("RGB")
-        buffer = BytesIO()
-        img.save(buffer, format="PNG")
-        buffer.seek(0)
-        sprite_images_bytes.append(buffer)
-    
+        try:
+            raw = b64.split(",", 1)[-1]
+            img = Image.open(BytesIO(base64.b64decode(raw))).convert("RGB")
+            buf = BytesIO()
+            img.save(buf, format="PNG")
+            buf.seek(0)
+            sprite_images_bytes.append(buf)
+        except Exception as e:
+            logger.warning(f"Skipping bad sprite base64: {e}")
+
+
+
     # =========================================
     #  Build the list of all candidate images
     # =========================================
@@ -3458,21 +3490,21 @@ SPRITE_DIR/"Zebra.sprite3"/"f3e322a25b9f79801066056de6f33fb1.png"
     #     feats = clip_embd.embed_image([buf])[0]  # extract CLIP embedding
     #     sprite_features.append(feats)
 
-    sprite_features = []
-    for sprite in sprites_data.values():  # assuming dict like {"Sprite 1": {...}, ...}
-        b64 = sprite["base64"]
-        if "," in b64:
-            b64 = b64.split(",", 1)[1]
+    # sprite_features = []
+    # for sprite in sprites_data.values():  # assuming dict like {"Sprite 1": {...}, ...}
+    #     b64 = sprite["base64"]
+    #     if "," in b64:
+    #         b64 = b64.split(",", 1)[1]
     
-        img_bytes = base64.b64decode(b64)
-        pil_img = Image.open(BytesIO(img_bytes)).convert("RGB")
+    #     img_bytes = base64.b64decode(b64)
+    #     pil_img = Image.open(BytesIO(img_bytes)).convert("RGB")
     
-        buf = BytesIO()
-        pil_img.save(buf, format="PNG")
-        buf.seek(0)
+    #     buf = BytesIO()
+    #     pil_img.save(buf, format="PNG")
+    #     buf.seek(0)
     
-        feats = clip_embd.embed_image([buf])[0]
-        sprite_features.append(feats)
+    #     feats = clip_embd.embed_image([buf])[0]
+    #     sprite_features.append(feats)
     
     # sprite_matrix = np.array(sprite_features, dtype=np.float32) 
     # # ============================== #
@@ -3507,29 +3539,97 @@ SPRITE_DIR/"Zebra.sprite3"/"f3e322a25b9f79801066056de6f33fb1.png"
     # img_matrix = l2_normalize_rows(img_matrix)
     # sprite_features = clip_embd.embed_image(sprite_base_path)
 
+    # # ============================== #
+    # #     COMPUTE SIMILARITIES   V1    #
+    # # ============================== #
+    # with open(f"{BLOCKS_DIR}/openclip_embeddings.json", "r") as f:
+    #     embedding_json = json.load(f)
+    #     # print(f"\n\n EMBEDDING JSON: {embedding_json}")
+        
+    # img_matrix = np.array([img["embeddings"] for img in embedding_json])
+    # sprite_matrix = np.array(sprite_features)
+    
+    # similarity = np.matmul(sprite_matrix, img_matrix.T)
+    # most_similar_indices = np.argmax(similarity, axis=1)
+
+    # # =========================================
+    # #  Compute similarities & pick best match
+    # # =========================================
+    # similarity = np.matmul(sprite_matrix, img_matrix.T)
+    # most_similar_indices = np.argmax(similarity, axis=1)
+
+    # # =========================================
+    # #  Copy matched sprite assets + collect data
+    # # =========================================
+    # project_data   = []
+    # copied_folders = set()
+
+
+        # ==============================
+    #   EMBED SPRITES (write to disk then embed)
+    # ==============================
+    # Ensure sprite_base_path exists and is empty (or a temp dir)
+    os.makedirs(sprite_base_path, exist_ok=True)
+    # clear existing files in sprite_base_path (careful in prod)
+    for f in os.listdir(sprite_base_path):
+        try:
+            os.remove(os.path.join(sprite_base_path, f))
+        except Exception:
+            pass
+
+    # Save the decoded BytesIO images as files into sprite_base_path
+    sprite_image_paths = []
+    for i, buf in enumerate(sprite_images_bytes):
+        img_path = os.path.join(sprite_base_path, f"sprite_{i}.png")
+        with open(img_path, "wb") as wf:
+            wf.write(buf.getvalue())
+        sprite_image_paths.append(os.path.normpath(img_path))
+
+    # sanity: if no sprites decoded, bail
+    if not sprite_image_paths:
+        raise RuntimeError("No valid sprite images to embed (sprite_image_paths empty).")
+
+    # Initialize embedder (you have a global clip_embd in your file; ensure it's initialized)
+    # clip_embd = OpenCLIPEmbeddings()  # if not already global
+
+    # Embed sprites (pass list of file paths)
+    try:
+        sprite_features = clip_embd.embed_image(sprite_image_paths)
+    except Exception as e:
+        logger.error(f"clip_embd.embed_image failed for sprite_image_paths: {e}")
+        raise
+
     # ============================== #
     #     COMPUTE SIMILARITIES       #
     # ============================== #
     with open(f"{BLOCKS_DIR}/openclip_embeddings.json", "r") as f:
         embedding_json = json.load(f)
-        # print(f"\n\n EMBEDDING JSON: {embedding_json}")
-        
-    img_matrix = np.array([img["embeddings"] for img in embedding_json])
-    sprite_matrix = np.array(sprite_features)
-    
-    similarity = np.matmul(sprite_matrix, img_matrix.T)
-    most_similar_indices = np.argmax(similarity, axis=1)
 
-    # =========================================
-    #  Compute similarities & pick best match
-    # =========================================
-    similarity = np.matmul(sprite_matrix, img_matrix.T)
+    img_matrix = np.array([img["embeddings"] for img in embedding_json], dtype=np.float32)
+    sprite_matrix = np.array(sprite_features, dtype=np.float32)
+
+    # sanity checks
+    if sprite_matrix.size == 0:
+        raise RuntimeError("No sprite embeddings (sprite_matrix empty).")
+    if img_matrix.size == 0:
+        raise RuntimeError("No reference embeddings (img_matrix empty).")
+    if sprite_matrix.shape[1] != img_matrix.shape[1]:
+        raise RuntimeError(
+            f"Embedding dimensionality mismatch: sprite {sprite_matrix.shape} vs img {img_matrix.shape}"
+        )
+
+    try:
+        similarity = np.matmul(sprite_matrix, img_matrix.T)
+    except Exception as e:
+        logger.error(f"Failed to compute similarity matrix: {e}")
+        raise
+
     most_similar_indices = np.argmax(similarity, axis=1)
 
     # =========================================
     #  Copy matched sprite assets + collect data
     # =========================================
-    project_data   = []
+    project_data = []
     copied_folders = set()
 
     for sprite_idx, matched_idx in enumerate(most_similar_indices):