Update app_main.py

app_main.py

@@ -235,6 +235,171 @@ def extract_images_from_pdf(pdf_path, output_json_path):
     except Exception as e:
         raise RuntimeError(f"❌ Error in extract_images_from_pdf: {str(e)}")
 
+def similarity_matching(input_json_path: str) -> str:
+    import uuid, shutil, tempfile
+    from langchain_experimental.open_clip.open_clip import OpenCLIPEmbeddings
+    from matplotlib.offsetbox import OffsetImage, AnnotationBbox
+    from io import BytesIO
+
+    logger.info("🔍 Running similarity matching...")
+
+    # ============================== #
+    #         DEFINE PATHS           #
+    # ============================== #
+    backdrop_images_path = r"E:\Pratham\2025\Harsh Sir\Scratch Vision\images\Backdrops"
+    sprite_images_path = r"E:\Pratham\2025\Harsh Sir\Scratch Vision\images\sprites"
+    image_dirs = [backdrop_images_path, sprite_images_path]
+    
+    
+    # ================================================= #
+    #   Generate Random UUID for project folder name    #
+    # ================================================= #
+    random_id = str(uuid.uuid4()).replace('-', '')
+    project_folder = os.path.join("outputs", f"project_{random_id}")
+    
+    # =========================================================================== #
+    #           Create empty json in project_{random_id} folder                   #
+    # =========================================================================== #
+    os.makedirs(project_folder, exist_ok=True)
+    project_json_path = os.path.join(project_folder, "project.json")
+
+    # ============================== #
+    #      READ SPRITE METADATA      #
+    # ============================== #
+    with open(input_json_path, 'r') as f:
+        sprites_data = json.load(f)
+
+    sprite_ids, texts, sprite_base64 = [], [], []
+    for sid, sprite in sprites_data.items():
+        sprite_ids.append(sid)
+        texts.append("This is " + sprite.get("description", sprite.get("name", "")))
+        sprite_base64.append(sprite["base64"])
+
+    # ========================================= #
+    #  Walk folders to collect all image paths  #
+    # ========================================= #
+    folder_image_paths = []
+    for image_dir in image_dirs:
+        for root, dirs, files in os.walk(image_dir):
+            for fname in files:
+                if fname.lower().endswith((".png", ".jpg", ".jpeg")):
+                    folder_image_paths.append(os.path.join(root, fname))
+
+    # ============================== #
+    #     INITIALIZE CLIP EMBEDDER   #
+    # ============================== #
+    clip_embd = OpenCLIPEmbeddings()
+    
+    # ============================== #
+    #   EMBED FOLDER IMAGES (REF)    #
+    # ============================== #
+    img_features = clip_embd.embed_image(folder_image_paths)
+
+    # ============================== #
+    #      DECODE SPRITE IMAGES      #
+    # ============================== #
+    temp_dir = tempfile.mkdtemp()
+    sprite_image_paths = []
+    for idx, b64 in enumerate(sprite_base64):
+        image_data = base64.b64decode(b64.split(",")[-1])
+        img = Image.open(BytesIO(image_data)).convert("RGB")
+        temp_path = os.path.join(temp_dir, f"sprite_{idx}.png")
+        img.save(temp_path)
+        sprite_image_paths.append(temp_path)
+
+    # ============================== #
+    #      EMBED SPRITE IMAGES       #
+    # ============================== #
+    sprite_features = clip_embd.embed_image(sprite_image_paths)
+    
+    # ============================== #
+    #     COMPUTE SIMILARITIES       #
+    # ============================== #
+    similarity = np.matmul(np.array(sprite_features), np.array(img_features).T)
+    most_similar_indices = np.argmax(similarity, axis=1)
+
+    # ============= Match and copy ================
+    project_data, backdrop_data = [], []
+    copied_folders = set()
+    for sprite_idx, matched_idx in enumerate(most_similar_indices):
+        matched_image_path = os.path.normpath(folder_image_paths[matched_idx])
+        matched_folder = os.path.dirname(matched_image_path)
+        if matched_folder in copied_folders:
+            continue
+        copied_folders.add(matched_folder)
+
+        # Sprite
+        sprite_json_path = os.path.join(matched_folder, 'sprite.json')
+        if os.path.exists(sprite_json_path):
+            with open(sprite_json_path, 'r') as f:
+                sprite_data = json.load(f)
+                project_data.append(sprite_data)
+
+            for fname in os.listdir(matched_folder):
+                if fname not in {os.path.basename(matched_image_path), 'sprite.json'}:
+                    shutil.copy2(os.path.join(matched_folder, fname), project_folder)
+
+        # Backdrop
+        if matched_image_path.startswith(os.path.normpath(backdrop_images_path)):
+            backdrop_json_path = os.path.join(matched_folder, 'project.json')
+            if os.path.exists(backdrop_json_path):
+                with open(backdrop_json_path, 'r') as f:
+                    backdrop_json_data = json.load(f)
+                for target in backdrop_json_data.get("targets", []):
+                    if target.get("isStage"):
+                        backdrop_data.append(target)
+                for fname in os.listdir(matched_folder):
+                    if fname not in {os.path.basename(matched_image_path), 'project.json'}:
+                        shutil.copy2(os.path.join(matched_folder, fname), project_folder)
+
+    # Merge JSON structure
+    final_project = {
+        "targets": [],
+        "monitors": [],
+        "extensions": [],
+        "meta": {
+            "semver": "3.0.0",
+            "vm": "11.3.0",
+            "agent": "OpenAI ScratchVision Agent"
+        }
+    }
+
+    for sprite in project_data:
+        if not sprite.get("isStage", False):
+            final_project["targets"].append(sprite)
+
+    if backdrop_data:
+        all_costumes, sounds = [], []
+        for idx, bd in enumerate(backdrop_data):
+            all_costumes.extend(bd.get("costumes", []))
+            if idx == 0 and "sounds" in bd:
+                sounds = bd["sounds"]
+        final_project["targets"].append({
+            "isStage": True,
+            "name": "Stage",
+            "variables": {},
+            "lists": {},
+            "broadcasts": {},
+            "blocks": {},
+            "comments": {},
+            "currentCostume": 1 if len(all_costumes) > 1 else 0,
+            "costumes": all_costumes,
+            "sounds": sounds,
+            "volume": 100,
+            "layerOrder": 0,
+            "tempo": 60,
+            "videoTransparency": 50,
+            "videoState": "on",
+            "textToSpeechLanguage": None
+        })
+
+    with open(project_json_path, 'w') as f:
+        json.dump(final_project, f, indent=2)
+
+    logger.info(f"🎉 Final project saved: {project_json_path}")
+    return project_json_path
+
+
 @app.route('/')
 def index():
     return render_template('app_index.html')
@@ -264,12 +429,14 @@ def process_pdf():
         json_path = None
         output_path, result = extract_images_from_pdf(saved_pdf_path, json_path)
 
+        project_output = similarity_matching(output_path)
         logger.info("Received request to process PDF.")
 
         return jsonify({
             "message": "✅ PDF processed successfully",
             "output_json": output_path,
-            "sprites": result
+            "sprites": result,
+            "project_output_json": project_output
         })
     except Exception as e:
         logger.exception("❌ Failed to process PDF")