scripts/triplane_rec_obaverse_train.py

"""
Train a diffusion model on images.
"""
import sys
import os
sys.path.append('.')

import torch as th
import torch.multiprocessing as mp

import argparse
import dnnlib
from guided_diffusion import dist_util, logger
from guided_diffusion.script_util import (
    args_to_dict,
    add_dict_to_argparser,
)
from nsr.train_util import TrainLoop3DTriplaneRec as TrainLoop
from nsr.script_util import create_3DAE_model, encoder_and_nsr_defaults, create_Triplane, loss_defaults
from datasets.g_buffer_objaverse import load_data, load_memory_data
from nsr.losses.builder import E3DGELossClass

from pdb import set_trace as st

from nsr.script_util import create_3DAE_model, encoder_and_nsr_defaults, loss_defaults, rendering_options_defaults, eg3d_options_default

import random
import json
import sys
import os

sys.path.append('.')
import torch.distributed as dist

import traceback


# def training_loop(rank, master_addr, master_port, args):
def training_loop(args):
    # dist_util.setup_dist(rank, master_addr, master_port, args.gpus)
    dist_util.setup_dist(args)

    # logger.configure(dir=args.logdir, format_strs=["tensorboard", "csv"])
    logger.configure(dir=args.logdir)

    # print(args)
    with open(os.path.join(args.logdir, 'args.json'), 'w') as f:
        json.dump(vars(args), f, indent=2)

    logger.log("creating data loader...")
    # TODO, load shapenet data
    # data = load_data(
    data = load_memory_data(
        file_path=args.data_dir,
        batch_size=args.batch_size,
        reso=args.image_size,
        reso_encoder=args.image_size_encoder, # 224 -> 128
        num_workers=args.num_workers,
        load_depth=args.depth_lambda > 0
        # load_depth=True # for evaluation
    )
    eval_data = data
    # eval_data = load_eval_data(
    #     file_path=args.data_dir,
    #     batch_size=args.eval_batch_size,
    #     reso=args.image_size,
    #     reso_encoder=args.image_size_encoder, # 224 -> 128
    #     num_workers=args.num_workers,
    #     load_depth=True # for evaluation
    # )
    # try dry run
    # batch = next(data)
    # batch = None

    # logger.log("creating model and diffusion...")
    logger.log("creating encoder and NSR decoder...")

    
    auto_encoder = create_Triplane( # basically overfitting tirplane
        **args_to_dict(args,
                       encoder_and_nsr_defaults().keys()))

    # auto_encoder = create_3DAE_model(
    #     **args_to_dict(args,
    #                    encoder_and_nsr_defaults().keys()))
    auto_encoder.to(dist_util.dev())

    # schedule_sampler = create_named_schedule_sampler(args.schedule_sampler, diffusion)


    opt = dnnlib.EasyDict(args_to_dict(args, loss_defaults().keys()) )
    loss_class = E3DGELossClass(dist_util.dev(), opt).to(dist_util.dev())

    logger.log("training...")
    TrainLoop(
        rec_model=auto_encoder,
        # encoder,
        # decoder
        loss_class=loss_class,
        # diffusion=diffusion,
        data=data,
        eval_data=eval_data,
        # data=batch,
        batch_size=args.batch_size,
        microbatch=args.microbatch,
        lr=args.lr,
        ema_rate=args.ema_rate,
        log_interval=args.log_interval,
        save_interval=args.save_interval,
        resume_checkpoint=args.resume_checkpoint,
        use_fp16=args.use_fp16,
        fp16_scale_growth=args.fp16_scale_growth,
        weight_decay=args.weight_decay,
        lr_anneal_steps=args.lr_anneal_steps,
        eval_interval=args.eval_interval,
    ).run_loop() # ! overfitting


def create_argparser(**kwargs):
    # defaults.update(model_and_diffusion_defaults())

    defaults = dict(
        num_workers=4,
        local_rank=0,
        gpus=1,
        image_size=128,
        image_size_encoder=224,
        iterations=150000,
        anneal_lr=False,
        lr=5e-5,
        weight_decay=0.0,
        lr_anneal_steps=0,
        batch_size=1,
        eval_batch_size=8,
        microbatch=-1,  # -1 disables microbatches
        ema_rate="0.9999",  # comma-separated list of EMA values
        log_interval=10,
        save_interval=10000,
        resume_checkpoint="",
        use_fp16=False,
        fp16_scale_growth=1e-3,
        data_dir="",
        # load_depth=False, # TODO
        logdir="/mnt/lustre/yslan/logs/nips23/",
        eval_interval=2500,
    )

    defaults.update(encoder_and_nsr_defaults())  # type: ignore
    defaults.update(loss_defaults())

    parser = argparse.ArgumentParser()
    add_dict_to_argparser(parser, defaults)

    return parser


if __name__ == "__main__":
    # os.environ[
    #     "TORCH_DISTRIBUTED_DEBUG"
    # ] = "DETAIL"  # set to DETAIL for runtime logging.
    # os.environ["TORCH_CPP_LOG_LEVEL"]="INFO"

    args = create_argparser().parse_args()
    args.local_rank = int(os.environ["LOCAL_RANK"])
    args.gpus = th.cuda.device_count()

    opts = args

    args.rendering_kwargs = rendering_options_defaults(opts)

    # Launch processes.
    print('Launching processes...')

    try:
        training_loop(args)
    # except KeyboardInterrupt as e:
    except Exception as e:
        # print(e)
        traceback.print_exc()
        dist_util.cleanup() # clean port and socket when ctrl+c