utils/depth_estimation.py

# utils/depth_estimation.py
import os
import torch
import numpy as np
from PIL import Image
import open3d as o3d
from transformers import DPTImageProcessor, DPTForDepthEstimation
from pathlib import Path
import logging
logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
from utils.image_utils import (
    resize_image_with_aspect_ratio
)
from utils.constants import TMPDIR
from easydict import EasyDict as edict


# Load models once during module import
image_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large")
depth_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large", ignore_mismatched_sizes=True)

def estimate_depth(image):
    # Ensure image is in RGB mode
    if image.mode != "RGB":
        image = image.convert("RGB")

    # Resize the image for the model
    image_resized = image.resize(
        (image.width, image.height),
        Image.Resampling.LANCZOS
    )

    # Prepare image for the model
    encoding = image_processor(image_resized, return_tensors="pt")

    # Forward pass
    with torch.no_grad():
        outputs = depth_model(**encoding)
        predicted_depth = outputs.predicted_depth

    # Interpolate to original size
    prediction = torch.nn.functional.interpolate(
        predicted_depth.unsqueeze(1),
        size=(image.height, image.width),
        mode="bicubic",
        align_corners=False,
    ).squeeze()

    # Convert to depth image
    output = prediction.cpu().numpy()
    depth_min = output.min()
    depth_max = output.max()
    max_val = (2**8) - 1

    # Normalize and convert to 8-bit image
    depth_image = max_val * (output - depth_min) / (depth_max - depth_min)
    depth_image = depth_image.astype("uint8")

    depth_pil = Image.fromarray(depth_image)

    return depth_pil, output

def create_3d_model(rgb_image, depth_array, voxel_size_factor=0.01):
    depth_o3d = o3d.geometry.Image(depth_array.astype(np.float32))
    rgb_o3d = o3d.geometry.Image(np.array(rgb_image))

    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
        rgb_o3d,
        depth_o3d,
        convert_rgb_to_intensity=False
    )

    # Create a point cloud from the RGBD image
    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(
        rgb_image.width,
        rgb_image.height,
        fx=1.0,
        fy=1.0,
        cx=rgb_image.width / 2.0,
        cy=rgb_image.height / 2.0,
    )

    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
        rgbd_image,
        camera_intrinsic
    )

    # Voxel downsample
    voxel_size = max(pcd.get_max_bound() - pcd.get_min_bound()) * voxel_size_factor
    voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(pcd, voxel_size=voxel_size)

    # Save the 3D model to a temporary file
    temp_dir = Path.cwd() / "temp_models"
    temp_dir.mkdir(exist_ok=True)
    model_path = temp_dir / "model.ply"
    o3d.io.write_voxel_grid(str(model_path), voxel_grid)

    return str(model_path)

def generate_depth_and_3d(input_image_path, voxel_size_factor):
    image = Image.open(input_image_path).convert("RGB")
    resized_image = resize_image_with_aspect_ratio(image, 2688, 1680)
    depth_image, depth_array = estimate_depth(resized_image)
    model_path = create_3d_model(resized_image, depth_array, voxel_size_factor=voxel_size_factor)
    return depth_image, model_path

def generate_depth_button_click(depth_image_source, voxel_size_factor, input_image, output_image, overlay_image, bordered_image_output):
    if depth_image_source == "Input Image":
        image_path = input_image
    elif depth_image_source == "Output Image":
        image_path = output_image
    elif depth_image_source == "Image with Margins":
        image_path = bordered_image_output
    else:
        image_path = overlay_image

    return generate_depth_and_3d(image_path, voxel_size_factor)

def create_3d_obj(rgb_image, raw_depth, image_path, depth=10, z_scale=200):
    """
    Creates a 3D object from RGB and depth images.

    Args:
        rgb_image (np.ndarray): The RGB image as a NumPy array.
        raw_depth (np.ndarray): The raw depth data.
        image_path (Path): The path to the original image.
        depth (int, optional): Depth parameter for Poisson reconstruction. Defaults to 10.
        z_scale (float, optional): Scaling factor for the Z-axis. Defaults to 200.

    Returns:
        str: The file path to the saved GLTF model.
    """
    # Normalize the depth image
    depth_image = ((raw_depth - raw_depth.min()) / (raw_depth.max() - raw_depth.min()) * 255).astype("uint8")
    depth_o3d = o3d.geometry.Image(depth_image)
    image_o3d = o3d.geometry.Image(rgb_image)

    # Create RGBD image
    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
        image_o3d, depth_o3d, convert_rgb_to_intensity=False
    )

    height, width = depth_image.shape

    # Define camera intrinsics
    camera_intrinsic = o3d.camera.PinholeCameraIntrinsic(
        width,
        height,
        fx=z_scale,
        fy=z_scale,
        cx=width / 2.0,
        cy=height / 2.0,
    )

    # Generate point cloud from RGBD image
    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic)

    # Scale the Z dimension
    points = np.asarray(pcd.points)
    depth_scaled = ((raw_depth - raw_depth.min()) / (raw_depth.max() - raw_depth.min())) * (z_scale*100)
    z_values = depth_scaled.flatten()[:len(points)]
    points[:, 2] *= z_values
    pcd.points = o3d.utility.Vector3dVector(points)

    # Estimate and orient normals
    pcd.estimate_normals(
        search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=60)
    )
    pcd.orient_normals_towards_camera_location(camera_location=np.array([0.0, 0.0, 1.5 ]))

    # Apply transformations
    pcd.transform([[1, 0, 0, 0],
                   [0, -1, 0, 0],
                   [0, 0, -1, 0],
                   [0, 0, 0, 1]])
    pcd.transform([[-1, 0, 0, 0],
                   [0, 1, 0, 0],
                   [0, 0, 1, 0],
                   [0, 0, 0, 1]])

    # Perform Poisson surface reconstruction
    print(f"Running Poisson surface reconstruction with depth {depth}")
    mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
        pcd, depth=depth, width=0, scale=1.1, linear_fit=True
    )
    print(f"Raw mesh vertices: {len(mesh_raw.vertices)}, triangles: {len(mesh_raw.triangles)}")

    # Simplify the mesh using vertex clustering
    voxel_size = max(mesh_raw.get_max_bound() - mesh_raw.get_min_bound()) / (max(width, height) * 0.8)
    mesh = mesh_raw.simplify_vertex_clustering(
        voxel_size=voxel_size,
        contraction=o3d.geometry.SimplificationContraction.Average,
    )
    print(f"Simplified mesh vertices: {len(mesh.vertices)}, triangles: {len(mesh.triangles)}")

    # Crop the mesh to the bounding box of the point cloud
    bbox = pcd.get_axis_aligned_bounding_box()
    mesh_crop = mesh.crop(bbox)

    # Save the mesh as a GLTF file
    temp_dir = Path.cwd() / "models"
    temp_dir.mkdir(exist_ok=True)
    gltf_path = str(temp_dir / f"{image_path.stem}.gltf")
    o3d.io.write_triangle_mesh(gltf_path, mesh_crop, write_triangle_uvs=True)
    return gltf_path
def depth_process_image(image_path, resized_width=800, z_scale=208):
    """
    Processes the input image to generate a depth map and a 3D mesh reconstruction.

    Args:
        image_path (str): The file path to the input image.

    Returns:
        list: A list containing the depth image, 3D mesh reconstruction, and GLTF file path.
    """
    image_path = Path(image_path)
    if not image_path.exists():
        raise ValueError("Image file not found")

    # Load and resize the image
    image_raw = Image.open(image_path).convert("RGB")
    print(f"Original size: {image_raw.size}")
    resized_height = int(resized_width * image_raw.size[1] / image_raw.size[0])
    image = image_raw.resize((resized_width, resized_height), Image.Resampling.LANCZOS)
    print(f"Resized size: {image.size}")

    # Prepare image for the model
    encoding = image_processor(image, return_tensors="pt")

    # Perform depth estimation
    with torch.no_grad():
        outputs = depth_model(**encoding)
        predicted_depth = outputs.predicted_depth

    # Interpolate depth to match the image size
    prediction = torch.nn.functional.interpolate(
        predicted_depth.unsqueeze(1),
        size=(image.height, image.width),
        mode="bicubic",
        align_corners=False,
    ).squeeze()

    # Normalize the depth image to 8-bit
    if torch.cuda.is_available():
        prediction = prediction.numpy()
    else:
        prediction = prediction.cpu().numpy()
    depth_min, depth_max = prediction.min(), prediction.max()
    depth_image = ((prediction - depth_min) / (depth_max - depth_min) * 255).astype("uint8")

    try:
        gltf_path = create_3d_obj(np.array(image), prediction, image_path, depth=10, z_scale=z_scale)
    except Exception:
        gltf_path = create_3d_obj(np.array(image), prediction, image_path, depth=8, z_scale=z_scale)

    img = Image.fromarray(depth_image)

    if torch.cuda.is_available():
        torch.cuda.empty_cache()
        torch.cuda.ipc_collect()
    return [img, gltf_path, gltf_path]

def get_depth_map_from_state(state, image_height=1024, image_width=1024):
    from diff_gaussian_rasterization import GaussianRasterizer, GaussianRasterizationSettings

    settings = GaussianRasterizationSettings(image_height=image_height, image_width=image_width, kernel_size=0.01,bg=(0.0, 0.0, 0.0))
    rasterizer = GaussianRasterizer(settings)
    # Assume state has necessary data like means3D, scales, etc.
    rendered_image, rendered_depth, _, _, _, _ = rasterizer(means3D=state["means3D"], means2D=state["means2D"], shs=state["shs"], colors_precomp=state["colors_precomp"], opacities=state["opacities"], scales=state["scales"], rotations=state["rotations"], cov3D_precomp=state["cov3D_precomp"])
    name = state['name']
    file_path = os.path.join(TMPDIR, f'{name}.png')

    depth_np = rendered_depth.cpu().numpy()[0]
    depth_min = depth_np.min()
    depth_max = depth_np.max()
    depth_np = (depth_np - depth_min) / (depth_max - depth_min) * 255
    depth_np = depth_np.astype(np.uint8)

    Image.fromarray(depth_np).save(file_path)
    return file_path