@CastleImitation Hi, thank you for making a new thread. Can you share the minimal code that I can reproduce your issue?

from d3rlpy.

Thank you sir!

Below is my customized neural network. Basically the input is 6*6 matrix, which is the patients' data such as age heights, test results, etc.

import torch
import torch.nn as nn
import torch.nn.functional as F
import dataclasses

class CustomEncoder(nn.Module):
def init(self, observation_shape, feature_size):
super().init()
print(f"Initializing CustomEncoder with observation shape: {observation_shape}")
self.feature_size = feature_size

    # 假设每个向量是1*6，我们有六个这样的向量
    embed_dim = 6  # 每个向量的维度
    seq_len = 6  # 向量的数量

    # 自注意力机制层的头数设置为2
    self.self_attention = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=2, batch_first=True)

    # 卷积层定义
    self.conv_layers = nn.ModuleList([
        nn.Conv1d(in_channels=1, out_channels=feature_size, kernel_size=3, padding=1),
        *[
            nn.Conv1d(in_channels=feature_size, out_channels=feature_size, kernel_size=3, padding=1)
            for _ in range(1, 200)
        ]
    ])

    self.direct_conv_layers = nn.ModuleList([
        nn.Conv1d(in_channels=1, out_channels=feature_size, kernel_size=3, padding=1),
        *[
            nn.Conv1d(in_channels=feature_size, out_channels=feature_size, kernel_size=3, padding=1)
            for _ in range(1, 150)
        ]
    ])

    self.dropout = nn.Dropout(0.2)

def forward(self, x):
    # 调整x的维度以适应自注意力层的输入要求
    # 假设输入x的形状已经是(batch_size, seq_len, embed_dim)，其中seq_len=6, embed_dim=6
    attn_output, _ = self.self_attention(x, x, x)  # 应用自注意力机制

    # 转换 attn_output 以适用于 Conv1d
    attn_output = attn_output.transpose(1, 2)  # 转置至(batch_size, embed_dim, seq_len)

    # 依次通过200个卷积层
    for conv in self.conv_layers:
        attn_output = F.relu(conv(attn_output))
    attn_output = self.dropout(attn_output)

    # 转换原始输入x以适用于 Conv1d
    original_x = x.transpose(1, 2)  # 转置至(batch_size, embed_dim, seq_len)
    for conv in self.direct_conv_layers:
        original_x = F.relu(conv(original_x))

    # 残差连接并通过ReLU激活
    output = F.relu(attn_output + original_x)

    return output.transpose(1, 2)  # 如果需要，转置回(batch_size, seq_len, embed_dim)

以下是工厂类，没有变化

@dataclasses.dataclass
class CustomEncoderFactory(d3rlpy.models.EncoderFactory):
feature_size: int

def create(self, observation_shape):
    return CustomEncoder(observation_shape, self.feature_size)

@staticmethod
def get_type() -> str:
    return "custom"

And I'm going to use this customized neural network on DQN BCQ and CQL RL algorithms.
But I got following error:
C:\Users\lifen\anaconda3\envs\DRL\python.exe C:\Users\lifen\OneDrive\桌面\DeepVent_Mod02\training\find_dqn.py
2024-02-12 10:44.16 [info ] Signatures have been automatically determined. action_signature=Signature(dtype=[dtype('int32')], shape=[(1,)]) observation_signature=Signature(dtype=[dtype('float64')], shape=[(6, 6)]) reward_signature=Signature(dtype=[dtype('float64')], shape=[(1,)])
2024-02-12 10:44.16 [info ] Action-space has been automatically determined. action_space=<ActionSpace.DISCRETE: 2>
2024-02-12 10:44.16 [info ] Action size has been automatically determined. action_size=343
2024-02-12 10:44.17 [info ] Signatures have been automatically determined. action_signature=Signature(dtype=[dtype('int32')], shape=[(1,)]) observation_signature=Signature(dtype=[dtype('float64')], shape=[(6, 6)]) reward_signature=Signature(dtype=[dtype('float64')], shape=[(1,)])
2024-02-12 10:44.17 [info ] Action-space has been automatically determined. action_space=<ActionSpace.DISCRETE: 2>
2024-02-12 10:44.17 [info ] Action size has been automatically determined. action_size=343
2024-02-12 10:44.17 [info ] Signatures have been automatically determined. action_signature=Signature(dtype=[dtype('int32')], shape=[(1,)]) observation_signature=Signature(dtype=[dtype('float64')], shape=[(6, 6)]) reward_signature=Signature(dtype=[dtype('float64')], shape=[(1,)])
2024-02-12 10:44.17 [info ] Action-space has been automatically determined. action_space=<ActionSpace.DISCRETE: 2>
2024-02-12 10:44.17 [info ] Action size has been automatically determined. action_size=343
2024-02-12 10:44.17 [info ] dataset info dataset_info=DatasetInfo(observation_signature=Signature(dtype=[dtype('float64')], shape=[(6, 6)]), action_signature=Signature(dtype=[dtype('int32')], shape=[(1,)]), reward_signature=Signature(dtype=[dtype('float64')], shape=[(1,)]), action_space=<ActionSpace.DISCRETE: 2>, action_size=343)
2024-02-12 10:44.17 [debug ] Building models...
Initializing CustomEncoder with observation shape: (6, 6)
Traceback (most recent call last):
File "C:\Users\lifen\OneDrive\桌面\DeepVent_Mod02\training\find_dqn.py", line 15, in
train(
File "C:\Users\lifen\OneDrive\桌面\DeepVent_Mod02\training\train_utils.py", line 78, in train
model.fit(
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\algos\qlearning\base.py", line 412, in fit
results = list(
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\algos\qlearning\base.py", line 501, in fitter
self.create_impl(observation_shape, action_size)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\base.py", line 311, in create_impl
self.inner_create_impl(observation_shape, action_size)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\algos\qlearning\dqn.py", line 171, in inner_create_impl
q_funcs, forwarder = create_discrete_q_function(
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\models\builders.py", line 66, in create_discrete_q_function
hidden_size = compute_output_size([observation_shape], encoder)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\d3rlpy\models\torch\encoders.py", line 276, in compute_output_size
y = encoder(*inputs)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "C:\Users\lifen\OneDrive\桌面\DeepVent_Mod02\utils\Cstm_Att_Conv.py", line 215, in forward
attn_output = F.relu(conv(attn_output))
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\conv.py", line 310, in forward
return self._conv_forward(input, self.weight, self.bias)
File "C:\Users\lifen\anaconda3\envs\DRL\lib\site-packages\torch\nn\modules\conv.py", line 306, in _conv_forward
return F.conv1d(input, weight, bias, self.stride,
RuntimeError: Given groups=1, weight of size [64, 1, 3], expected input[1, 6, 6] to have 1 channels, but got 6 channels instead

Process finished with exit code 1

from d3rlpy.

Sir,
Could it just input 66 text data as observation in our code in d3rlpy?
And in future, I want to increase the dimension of the observation from 66 to 466. 4 means the latest 4 time intervals' 6*6 observations. So can our code of d3rlpy support such observation_shape input?

Sincerily hope your kind reply!

from d3rlpy.

Thank you for sharing your code, but I found that it's simply that your encoder is not doing right and it's not an issue in d3rlpy side. The error is complaining that the input shape of 1d convolution layer has the wrong shape. What I can suggest here is to test your encoder without d3rlpy as follows:

encoder = CustomEncoder(observation_shape=(6, 6), feature_size=xxx)
test_input = torch.rand(1, 6, 6)
encoder(test_input)

from d3rlpy.

from d3rlpy.

Yes, d3rlpy supports arbitrary shapes as inputs.

from d3rlpy.

from d3rlpy.

This issue was about PyTorch usage rather than d3rlpy. So please let me close this issue. If you have any further discussion, feel free to reopen this issue.

from d3rlpy.