Huggingface版本是否提供lora微调的支持？ about cogvlm HOT 3 OPEN

import torch.nn as nn
import logging
import math


class LoRABlock(nn.Module):
    """
    A simple implementation of LoRA
    """
    def __init__(self, linear: nn.Linear, rank: int) -> None:
        super(LoRABlock, self).__init__()
        assert isinstance(linear, nn.Linear), "LoRA only supports Linear module!"
        linear_dtype = linear.weight.dtype
        input_dim = linear.weight.shape[-1]
        out_dim = linear.weight.shape[0]
        self.original_linear = nn.Linear(input_dim, out_dim, dtype=linear_dtype)
        self.original_linear.weight.data = linear.weight.data.clone().detach()
        if linear.bias is not None:
            self.original_linear.bias.data = linear.bias.data.clone().detach()
        else:
            self.original_linear.bias.data.zero_()
        self.original_linear.requires_grad_(False)

        rank_upper_bound = (input_dim * out_dim) / (input_dim + out_dim + 1)
        while rank > rank_upper_bound:
            rank = math.floor(rank/2)
            logging.warning("Preset rank ({}) was too high, degrading to {}".format(rank * 2, rank))
            if rank == 0:
                raise ValueError("rank_upper_bound error: current value: {}.\n \
                                 The cause of this issue is: input_dim: {}, out_dim:\
                                 {}".format(rank_upper_bound, input_dim, out_dim))

        assert rank <= rank_upper_bound, "Rank is too large to shrink the original model"
        
        self.B = nn.Linear(input_dim, rank, bias=False, dtype=linear_dtype)
        self.B.weight.data.random_()  
        self.A = nn.Linear(rank, out_dim, bias=False, dtype=linear_dtype)
        self.A.weight.data.zero_()

        self.weight = self.original_linear.weight
        self.bias = self.original_linear.bias
        
    def forward(self, x):
        origin_output = self.original_linear(x)
        lora_modification = self.A(self.B(x))
        return origin_output + lora_modification
    

def substitute_model_with_lora(model: nn.Module, rank: int=32):
    """
    replace all linear blocks in a pytorch Module
    """
    names = dir(model)
    for name in names:
        if not name.startswith("_") and not name.startswith("get") and name != "base_model":
            obj = getattr(model, name)
            if isinstance(obj, nn.Linear):
                lora_block = LoRABlock(obj, rank)
                setattr(model, name, lora_block)
            elif isinstance(obj, nn.Module) and not isinstance(obj, nn.ModuleList):
                if next(obj.named_parameters(), None) is not None:
                    lora_block = substitute_model_with_lora(obj, rank)
                    setattr(model, name, lora_block)
            elif isinstance(obj, nn.ModuleList):
                lora_list = nn.ModuleList()
                for sub_module in obj:
                    lora_sub_module = substitute_model_with_lora(sub_module, rank)
                    lora_list.append(lora_sub_module)
                setattr(model, name, lora_list)
            else:
                pass
    return model