corl-team / corl Goto Github PK

BUG
/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx:127:21: Cannot assign type 'void (const char ) except * nogil' to 'void ()(const char *) noexcept nogil'. Exception values are incompatible. Suggest adding 'noexcept' to type 'void (const char *) except * nogil'.
Traceback (most recent call last):
File "/home/fs01/mah4021/CORL/algorithms/offline/dt.py", line 11, in
import d4rl # noqa
File "/home/fs01/mah4021/d4rl/d4rl/init.py", line 14, in
import d4rl.locomotion
File "/home/fs01/mah4021/d4rl/d4rl/locomotion/init.py", line 2, in
from d4rl.locomotion import ant
File "/home/fs01/mah4021/d4rl/d4rl/locomotion/ant.py", line 20, in
import mujoco_py
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/init.py", line 2, in
from mujoco_py.builder import cymj, ignore_mujoco_warnings, functions, MujocoException
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 504, in
cymj = load_cython_ext(mujoco_path)
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 110, in load_cython_ext
cext_so_path = builder.build()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 226, in build
built_so_file_path = self._build_impl()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 296, in _build_impl
so_file_path = super()._build_impl()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 239, in _build_impl
dist.ext_modules = cythonize([self.extension])
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/Cython/Build/Dependencies.py", line 1154, in cythonize
cythonize_one(*args)
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/Cython/Build/Dependencies.py", line 1321, in cythonize_one
raise CompileError(None, pyx_file)
Cython.Compiler.Errors.CompileError: /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx
(CORL) bash-4.4$ pip instal --upgrade cython
ERROR: unknown command "instal" - maybe you meant "install"
(CORL) bash-4.4$ pip install --upgrade cython
Requirement already satisfied: cython in /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages (3.0.5)
(CORL) bash-4.4$ python dt.py
Compiling /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx because it changed.
[1/1] Cythonizing /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx
performance hint: /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx:67:5: Exception check on 'c_warning_callback' will always require the GIL to be acquired.
Possible solutions:
1. Declare the function as 'noexcept' if you control the definition and you're sure you don't want the function to raise exceptions.
2. Use an 'int' return type on the function to allow an error code to be returned.
performance hint: /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx:104:5: Exception check on 'c_error_callback' will always require the GIL to be acquired.
Possible solutions:
1. Declare the function as 'noexcept' if you control the definition and you're sure you don't want the function to raise exceptions.
2. Use an 'int' return type on the function to allow an error code to be returned.

Error compiling Cython file:

...
See c_warning_callback, which is the C wrapper to the user defined function
'''
global py_warning_callback
global mju_user_warning
py_warning_callback = warn
mju_user_warning = c_warning_callback
^

/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx:92:23: Cannot assign type 'void (const char ) except * nogil' to 'void ()(const char *) noexcept nogil'. Exception values are incompatible. Suggest adding 'noexcept' to type 'void (const char *) except * nogil'.

Error compiling Cython file:

...
See c_warning_callback, which is the C wrapper to the user defined function
'''
global py_error_callback
global mju_user_error
py_error_callback = err_callback
mju_user_error = c_error_callback
^

/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx:127:21: Cannot assign type 'void (const char ) except * nogil' to 'void ()(const char *) noexcept nogil'. Exception values are incompatible. Suggest adding 'noexcept' to type 'void (const char *) except * nogil'.
Traceback (most recent call last):
File "/home/fs01/mah4021/CORL/algorithms/offline/dt.py", line 11, in
import d4rl # noqa
File "/home/fs01/mah4021/d4rl/d4rl/init.py", line 14, in
import d4rl.locomotion
File "/home/fs01/mah4021/d4rl/d4rl/locomotion/init.py", line 2, in
from d4rl.locomotion import ant
File "/home/fs01/mah4021/d4rl/d4rl/locomotion/ant.py", line 20, in
import mujoco_py
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/init.py", line 2, in
from mujoco_py.builder import cymj, ignore_mujoco_warnings, functions, MujocoException
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 504, in
cymj = load_cython_ext(mujoco_path)
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 110, in load_cython_ext
cext_so_path = builder.build()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 226, in build
built_so_file_path = self._build_impl()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 296, in _build_impl
so_file_path = super()._build_impl()
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/builder.py", line 239, in _build_impl
dist.ext_modules = cythonize([self.extension])
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/Cython/Build/Dependencies.py", line 1154, in cythonize
cythonize_one(*args)
File "/home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/Cython/Build/Dependencies.py", line 1321, in cythonize_one
raise CompileError(None, pyx_file)
Cython.Compiler.Errors.CompileError: /home/fs01/mah4021/.conda/envs/CORL/lib/python3.10/site-packages/mujoco_py/cymj.pyx

FIX
(CORL) bash-4.4$ pip install "cython<3"
Collecting cython<3
Downloading Cython-0.29.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl.metadata (3.1 kB)
Downloading Cython-0.29.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (1.9 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.9/1.9 MB 11.4 MB/s eta 0:00:00
Installing collected packages: cython
Attempting uninstall: cython
Found existing installation: Cython 3.0.5
Uninstalling Cython-3.0.5:
Successfully uninstalled Cython-3.0.5
Successfully installed cython-0.29.36

How exactly do I do roll outs with the trained policy with rendering? Is that naively easy to do, or requires work?

There we will track out progress for Minari integration with CORL. Minari is a standard format for offline RL datasets, with popular reference datasets and related utilities, which we believe will replace D4RL in the future by combining most of the existing benchmarks under unified interface and storage. And we want to be prepared! Eventually this will become a CORLv2.

The plan is to add an experimental separate directory with algorithms using Minari and to retraing all algorithms on the D4RL datasets currently (and in the future) re-created in the Minari.

Adapted algorithms:

• BC
• TD3 + BC
• AWAC
• CQL
• IQL
• SAC-N
• EDAC
• DT

Retrained algorithms (only Adroit for now):

• BC
• TD3 + BC
• AWAC
• CQL
• IQL
• SAC-N
• EDAC
• DT

As shown in this project, there are large amount offline-to-online results of Antmaze domains, however, lacking gym-mujoco

The link to Katakomba in the readme leads to an archived repo

I have trained a CQL agent on HalfCheetah with a good performance, getting to 5000 per episode. I modified the code to save the best model during training not every mode after evaluation:

            if config.checkpoints_path:
                torch.save(
                    trainer.state_dict(),
                    os.path.join(config.checkpoints_path, f"checkpoint_{t}.pt"),
                )

to:

            if config.checkpoints_path and eval_score > best_eval_score:
                best_eval_score = eval_score
                torch.save(
                    trainer.state_dict(),
                    os.path.join(config.checkpoints_path, f"best.pt"),
                )

The issue happens when I load this model and run it over a newly created environment. In that case, the performance drops to -700 per episode which is basically the performance of a random agent. Here is the code to test the agent:

@torch.no_grad()
def test_actor(
    config: argparse
) -> np.ndarray:
    env = gym.make(config.env)

    state_dim = env.observation_space.shape[0]
    action_dim = env.action_space.shape[0]
    max_action = float(env.action_space.high[0])

    # Set seeds
    seed = config.seed
    set_seed(seed, env)

    critic_1 = FullyConnectedQFunction(
        state_dim,
        action_dim,
        config.orthogonal_init,
        config.q_n_hidden_layers,
    ).to(config.device)
    critic_2 = FullyConnectedQFunction(state_dim, action_dim, config.orthogonal_init).to(
        config.device
    )
    critic_1_optimizer = torch.optim.Adam(list(critic_1.parameters()), config.qf_lr)
    critic_2_optimizer = torch.optim.Adam(list(critic_2.parameters()), config.qf_lr)

    actor = TanhGaussianPolicy(
        state_dim,
        action_dim,
        max_action,
        log_std_multiplier=config.policy_log_std_multiplier,
        orthogonal_init=config.orthogonal_init,
    ).to(config.device)
    actor_optimizer = torch.optim.Adam(actor.parameters(), config.policy_lr)

    kwargs = {
        "critic_1": critic_1,
        "critic_2": critic_2,
        "critic_1_optimizer": critic_1_optimizer,
        "critic_2_optimizer": critic_2_optimizer,
        "actor": actor,
        "actor_optimizer": actor_optimizer,
        "discount": config.discount,
        "soft_target_update_rate": config.soft_target_update_rate,
        "device": config.device,
        # CQL
        "target_entropy": -np.prod(env.action_space.shape).item(),
        "alpha_multiplier": config.alpha_multiplier,
        "use_automatic_entropy_tuning": config.use_automatic_entropy_tuning,
        "backup_entropy": config.backup_entropy,
        "policy_lr": config.policy_lr,
        "qf_lr": config.qf_lr,
        "bc_steps": config.bc_steps,
        "target_update_period": config.target_update_period,
        "cql_n_actions": config.cql_n_actions,
        "cql_importance_sample": config.cql_importance_sample,
        "cql_lagrange": config.cql_lagrange,
        "cql_target_action_gap": config.cql_target_action_gap,
        "cql_temp": config.cql_temp,
        "cql_alpha": config.cql_alpha,
        "cql_max_target_backup": config.cql_max_target_backup,
        "cql_clip_diff_min": config.cql_clip_diff_min,
        "cql_clip_diff_max": config.cql_clip_diff_max,
    }

    print("---------------------------------------")
    print(f"TESTING CQL, Env: {config.env}, Seed: {seed}")
    print("---------------------------------------")

    # Initialize actor
    trainer = ContinuousCQL(**kwargs)

    model_path = os.path.join(config.checkpoints_path, f"best.pt")
    trainer.load_state_dict(torch.load(model_path, map_location=config.device))
    actor = trainer.actor

    actor.eval()
    episode_rewards = []
    for _ in range(config.test_episodes):
        state, done = env.reset(), False
        episode_reward = 0.0
        while not done:
            action = actor.act(state, config.device)
            state, reward, done, _ = env.step(action)
            episode_reward += reward
        episode_rewards.append(episode_reward)

    print(f"TEST Mean reward: {np.mean(episode_rewards)}")

Hi, in the github project main page, there are several tables with performance results comparing several algorithms.

Is there documentation from where these numbers como and why they mean ? As well as the methodology used ?

Thanks a lot !