d42a5fb354
## implementation I implemented HER solely as a replay buffer. It is done by temporarily directly re-writing transitions storage (`self._meta`) during the `sample_indices()` call. The original transitions are cached and will be restored at the beginning of the next sampling or when other methods is called. This will make sure that. for example, n-step return calculation can be done without altering the policy. There is also a problem with the original indices sampling. The sampled indices are not guaranteed to be from different episodes. So I decided to perform re-writing based on the episode. This guarantees that the sampled transitions from the same episode will have the same re-written goal. This also make the re-writing ratio calculation slightly differ from the paper, but it won't be too different if there are many episodes in the buffer. In the current commit, HER replay buffer only support 'future' strategy and online sampling. This is the best of HER in term of performance and memory efficiency. I also add a few more convenient replay buffers (`HERVectorReplayBuffer`, `HERReplayBufferManager`), test env (`MyGoalEnv`), gym wrapper (`TruncatedAsTerminated`), unit tests, and a simple example (examples/offline/fetch_her_ddpg.py). ## verification I have added unit tests for almost everything I have implemented. HER replay buffer was also tested using DDPG on [`FetchReach-v3` env](https://github.com/Farama-Foundation/Gymnasium-Robotics). I used default DDPG parameters from mujoco example and didn't tune anything further to get this good result! (train script: examples/offline/fetch_her_ddpg.py). 
92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from tianshou.data import (
|
|
HERReplayBuffer,
|
|
HERReplayBufferManager,
|
|
PrioritizedReplayBuffer,
|
|
PrioritizedReplayBufferManager,
|
|
ReplayBuffer,
|
|
ReplayBufferManager,
|
|
)
|
|
|
|
|
|
class VectorReplayBuffer(ReplayBufferManager):
|
|
"""VectorReplayBuffer contains n ReplayBuffer with the same size.
|
|
|
|
It is used for storing transition from different environments yet keeping the order
|
|
of time.
|
|
|
|
:param int total_size: the total size of VectorReplayBuffer.
|
|
:param int buffer_num: the number of ReplayBuffer it uses, which are under the same
|
|
configuration.
|
|
|
|
Other input arguments (stack_num/ignore_obs_next/save_only_last_obs/sample_avail)
|
|
are the same as :class:`~tianshou.data.ReplayBuffer`.
|
|
|
|
.. seealso::
|
|
|
|
Please refer to :class:`~tianshou.data.ReplayBuffer` for other APIs' usage.
|
|
"""
|
|
|
|
def __init__(self, total_size: int, buffer_num: int, **kwargs: Any) -> None:
|
|
assert buffer_num > 0
|
|
size = int(np.ceil(total_size / buffer_num))
|
|
buffer_list = [ReplayBuffer(size, **kwargs) for _ in range(buffer_num)]
|
|
super().__init__(buffer_list)
|
|
|
|
|
|
class PrioritizedVectorReplayBuffer(PrioritizedReplayBufferManager):
|
|
"""PrioritizedVectorReplayBuffer contains n PrioritizedReplayBuffer with same size.
|
|
|
|
It is used for storing transition from different environments yet keeping the order
|
|
of time.
|
|
|
|
:param int total_size: the total size of PrioritizedVectorReplayBuffer.
|
|
:param int buffer_num: the number of PrioritizedReplayBuffer it uses, which are
|
|
under the same configuration.
|
|
|
|
Other input arguments (alpha/beta/stack_num/ignore_obs_next/save_only_last_obs/
|
|
sample_avail) are the same as :class:`~tianshou.data.PrioritizedReplayBuffer`.
|
|
|
|
.. seealso::
|
|
|
|
Please refer to :class:`~tianshou.data.ReplayBuffer` for other APIs' usage.
|
|
"""
|
|
|
|
def __init__(self, total_size: int, buffer_num: int, **kwargs: Any) -> None:
|
|
assert buffer_num > 0
|
|
size = int(np.ceil(total_size / buffer_num))
|
|
buffer_list = [
|
|
PrioritizedReplayBuffer(size, **kwargs) for _ in range(buffer_num)
|
|
]
|
|
super().__init__(buffer_list)
|
|
|
|
def set_beta(self, beta: float) -> None:
|
|
for buffer in self.buffers:
|
|
buffer.set_beta(beta)
|
|
|
|
|
|
class HERVectorReplayBuffer(HERReplayBufferManager):
|
|
"""HERVectorReplayBuffer contains n HERReplayBuffer with same size.
|
|
|
|
It is used for storing transition from different environments yet keeping the order
|
|
of time.
|
|
|
|
:param int total_size: the total size of HERVectorReplayBuffer.
|
|
:param int buffer_num: the number of HERReplayBuffer it uses, which are
|
|
under the same configuration.
|
|
|
|
Other input arguments are the same as :class:`~tianshou.data.HERReplayBuffer`.
|
|
|
|
.. seealso::
|
|
Please refer to :class:`~tianshou.data.ReplayBuffer` for other APIs' usage.
|
|
"""
|
|
|
|
def __init__(self, total_size: int, buffer_num: int, **kwargs: Any) -> None:
|
|
assert buffer_num > 0
|
|
size = int(np.ceil(total_size / buffer_num))
|
|
buffer_list = [HERReplayBuffer(size, **kwargs) for _ in range(buffer_num)]
|
|
super().__init__(buffer_list)
|