remove fairseq

codeclm/tokenizer/Flow1dVAE/generate_septoken.py

@@ -14,8 +14,8 @@ import tools.torch_tools as torch_tools
 from safetensors.torch import load_file
 from third_party.demucs.models.pretrained import get_model_from_yaml
 from filelock import FileLock
-import kaldiio
-# os.path.join(args.model_dir, "htdemucs.pth"), os.path.join(args.model_dir, "htdemucs.yaml")
+
+
 class Separator:
     def __init__(self, dm_model_path='demucs/ckpt/htdemucs.pth', dm_config_path='demucs/ckpt/htdemucs.yaml', gpu_id=0) -> None:
         if torch.cuda.is_available() and gpu_id < torch.cuda.device_count():

codeclm/tokenizer/Flow1dVAE/model_1rvq.py

@@ -19,12 +19,11 @@ from libs.rvq.descript_quantize3 import ResidualVectorQuantize
 
 from models_gpt.models.gpt2_rope2_time_new_correct_mask_noncasual_reflow import GPT2Model
 from models_gpt.models.gpt2_config import GPT2Config
+from our_MERT_BESTRQ.mert_fairseq.models.musicfm.musicfm_model import MusicFMModel, MusicFMConfig
 
 from torch.cuda.amp import autocast
 
 
-from our_MERT_BESTRQ.test import load_model
-
 class HubertModelWithFinalProj(HubertModel):
     def __init__(self, config):
         super().__init__(config)
@@ -272,6 +271,7 @@ class PromptCondAudioDiffusion(nn.Module):
         ssl_layer=None,
         uncondition=True,
         out_paint=False,
+        ssl_path='ckpt/encode-s12k.pt'
     ):
         super().__init__()
 
@@ -294,28 +294,24 @@ class PromptCondAudioDiffusion(nn.Module):
         self.rsq48towav2vec = torchaudio.transforms.Resample(48000, 16000)
         # self.wav2vec = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True)
         # self.wav2vec_processor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True)
-        self.bestrq = load_model(
-            model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq',
-            checkpoint_dir='ckpt/encode-s12k.pt',
-        )
+        self.bestrq = MusicFMModel(MusicFMConfig())
         self.rsq48tobestrq = torchaudio.transforms.Resample(48000, 24000)
         self.rsq48tohubert = torchaudio.transforms.Resample(48000, 16000)
-        for v in self.bestrq.parameters():v.requires_grad = False
         self.rvq_bestrq_emb = ResidualVectorQuantize(input_dim = 1024, n_codebooks = 1, codebook_size = 16_384, codebook_dim = 32, quantizer_dropout = 0.0, stale_tolerance=200)
         for v in self.rvq_bestrq_emb.parameters():v.requires_grad = False
         # self.hubert = HubertModelWithFinalProj.from_pretrained("ckpt/models--lengyue233--content-vec-best/snapshots/c0b9ba13db21beaa4053faae94c102ebe326fd68")
         # for v in self.hubert.parameters():v.requires_grad = False
         self.zero_cond_embedding1 = nn.Parameter(torch.randn(32*32,))
         # self.xvecmodel = XVECModel()
-        config = GPT2Config(n_positions=1000,n_layer=39,n_head=30,n_embd=1200)
-        unet = GPT2Model(config)
-        mlp =  nn.Sequential(
-            nn.Linear(1200, 1024), 
-            nn.SiLU(),                  
-            nn.Linear(1024, 1024),      
-            nn.SiLU(),                 
-            nn.Linear(1024, 768)  
-        )
+        # config = GPT2Config(n_positions=1000,n_layer=39,n_head=30,n_embd=1200)
+        # unet = GPT2Model(config)
+        # mlp =  nn.Sequential(
+        #     nn.Linear(1200, 1024), 
+        #     nn.SiLU(),                  
+        #     nn.Linear(1024, 1024),      
+        #     nn.SiLU(),                 
+        #     nn.Linear(1024, 768)  
+        # )
         self.set_from = "random"
         # self.cfm_wrapper = BASECFM(unet, mlp,self.ssl_layer)
         self.mask_emb = torch.nn.Embedding(3, 48)
@@ -538,8 +534,6 @@ class PromptCondAudioDiffusion(nn.Module):
         input_audio_0 = self.preprocess_audio(input_audio_0)
         input_audio_1 = self.preprocess_audio(input_audio_1)
 
-        self.bestrq.eval()
-
         # bestrq_middle,bestrq_last = self.extract_bestrq_embeds(input_audios)
         # bestrq_middle = bestrq_middle.detach()
         # bestrq_last = bestrq_last.detach()
@@ -575,8 +569,6 @@ class PromptCondAudioDiffusion(nn.Module):
         input_audio_0 = self.preprocess_audio(input_audio_0)
         input_audio_1 = self.preprocess_audio(input_audio_1)
 
-        self.bestrq.eval()
-
         # bestrq_middle,bestrq_last = self.extract_bestrq_embeds(input_audios)
         # bestrq_middle = bestrq_middle.detach()
         # bestrq_last = bestrq_last.detach()

codeclm/tokenizer/Flow1dVAE/model_septoken.py

@@ -20,9 +20,9 @@ from libs.rvq.descript_quantize3 import ResidualVectorQuantize
 
 from models_gpt.models.gpt2_rope2_time_new_correct_mask_noncasual_reflow import GPT2Model
 from models_gpt.models.gpt2_config import GPT2Config
+from our_MERT_BESTRQ.mert_fairseq.models.musicfm.musicfm_model import MusicFMModel, MusicFMConfig
 
 from torch.cuda.amp import autocast
-from our_MERT_BESTRQ.test import load_model
 
 class HubertModelWithFinalProj(HubertModel):
     def __init__(self, config):
@@ -253,6 +253,7 @@ class PromptCondAudioDiffusion(nn.Module):
         snr_gamma=None,
         uncondition=True,
         out_paint=False,
+        ssl_path='ckpt/encode-s12k.pt'
     ):
         super().__init__()
 
@@ -273,13 +274,9 @@ class PromptCondAudioDiffusion(nn.Module):
         self.rsq48towav2vec = torchaudio.transforms.Resample(48000, 16000)
         # self.wav2vec = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True)
         # self.wav2vec_processor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True)
-        self.bestrq = load_model(
-            model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq',
-            checkpoint_dir='ckpt/encode-s12k.pt',
-        )
+        self.bestrq = MusicFMModel(MusicFMConfig())
         self.rsq48tobestrq = torchaudio.transforms.Resample(48000, 24000)
         self.rsq48tohubert = torchaudio.transforms.Resample(48000, 16000)
-        for v in self.bestrq.parameters():v.requires_grad = False
         self.rvq_bestrq_emb = ResidualVectorQuantize(input_dim = 1024, n_codebooks = 1, codebook_size = 16_384, codebook_dim = 32, quantizer_dropout = 0.0, stale_tolerance=200)
         self.rvq_bestrq_bgm_emb = ResidualVectorQuantize(input_dim = 1024, n_codebooks = 1, codebook_size = 16_384, codebook_dim = 32, quantizer_dropout = 0.0, stale_tolerance=200)
         # self.hubert = HubertModelWithFinalProj.from_pretrained("ckpt/models--lengyue233--content-vec-best/snapshots/c0b9ba13db21beaa4053faae94c102ebe326fd68")

codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq/models/musicfm/musicfm_model.py

@@ -4,14 +4,6 @@ except:
     import sys, os
     sys.path.append(os.path.dirname(os.path.abspath(__file__)))
     from model.musicfm_25hz import MusicFM25Hz
-try:
-    from fairseq.fairseq.dataclass import FairseqDataclass
-    from fairseq.fairseq.models import BaseFairseqModel, register_model
-    from fairseq.fairseq.tasks.fairseq_task import FairseqTask
-except:
-    from fairseq.dataclass import FairseqDataclass
-    from fairseq.models import BaseFairseqModel, register_model
-    from fairseq.tasks.fairseq_task import FairseqTask
     
 from dataclasses import dataclass, field
 from typing import List, Tuple, Optional
@@ -22,7 +14,7 @@ from logging import getLogger
 logger = getLogger(__name__)
 
 @dataclass
-class MusicFMConfig(FairseqDataclass):
+class MusicFMConfig:
     label_rate:int = field(default=25)
     num_codebooks:int = field(default=1)
     codebook_dim:int = field(default=16)
@@ -45,9 +37,8 @@ class MusicFMConfig(FairseqDataclass):
 
 SAMPLE_RATE = 24_000
 
-@register_model("musicfm", dataclass=MusicFMConfig)
-class MusicFMModel(BaseFairseqModel):
-    def __init__(self, cfg: MusicFMConfig, task_cfg: FairseqTask):
+class MusicFMModel(torch.nn.Module):
+    def __init__(self, cfg: MusicFMConfig):
         super().__init__()
         self.cfg = cfg
         self.model = MusicFM25Hz(
@@ -91,19 +82,3 @@ class MusicFMModel(BaseFairseqModel):
             result["logits"] = logits
             result["hidden_emb"] = hidden_emb
             return result
-
-    @classmethod
-    def build_model(cls, cfg: MusicFMConfig, task: FairseqTask):
-        """Build a new model instance."""
-
-        model = MusicFMModel(cfg, task.cfg)
-        import numpy as np
-        s = 0
-        for param in model.parameters():
-            s += np.product(param.size())
-        print('# of parameters: '+str(s/1024.0/1024.0))
-        return model
-
-    def get_losses(self, result, batch):
-        return result['losses']
-