pip install aft-pytorch

from aft_pytorch import AFTFull

layer = AFTFull(
    max_seqlen=20,
    dim=512,
    hidden_dim=64
)

# a batch of sequences with 10 timesteps of length 512 each
x = torch.rand(32, 10, 512)
y = layer(x) # [32, 10, 512]

from aft_pytorch import AFTSimple

layer = AFTSimple(
    max_seqlen=20,
    dim=512,
    hidden_dim=64
)

# a batch of sequences with 10 timesteps of length 512 each
x = torch.rand(32, 10, 512)
y = layer(x) # [32, 10, 512]

from aft_pytorch import AFTLocal

layer = AFTLocal(
    max_seqlen=20,
    dim=512,
    hidden_dim=64
)

# a batch of sequences with 10 timesteps of length 512 each
x = torch.rand(32, 10, 512)
y = layer(x) # [32, 10, 512]

@misc{attention-free-transformer,
title = {An Attention Free Transformer},
author = {Shuangfei Zhai and Walter Talbott and Nitish Srivastava and Chen Huang and Hanlin Goh and Ruixiang Zhang and Josh Susskind},
year = {2021},
URL = {https://arxiv.org/pdf/2105.14103.pdf}
}

Contributors

Stargazers

Forkers

dumpmemory fratenuta zolekode alihussainia muralits98 stjordanis neshomago zzwang058 datta0 trendingtechnology zshy1205 shadowkun peternara codewithflycat tuananh1007 icodein sumanthbajjuri07 crashmoon vincentami mhhamdan zlgenuine lihuibng

self.encoder_transformer = nn.ModuleList()
for _ in range(6):
    self.encoder_transformer.append(AFTFull(max_seqlen=500, dim=512,hidden_dim=256))

for _, layer in enumerate(self.encoder_transformer):`
    x = layer(x) + x