I tested the memory consumption of resnet 101 and the corresponding momentum version using the scripts below. However, difference in memory consumption between these 2 models is only a few hundreds MBs.
from momentumnet import transform_to_momentumnet
from torchvision.models import resnet101
resnet = resnet101(pretrained=True).to(0)
mresnet101 = transform_to_momentumnet(resnet, gamma=0.99998, use_backprop=False)
#mresnet101 = resnet101(pretrained=True).to(0)
optim = torch.optim.Adam(mresnet101.parameters(), lr=1e-4)
for i in range(100):
x = torch.randn(70, 3, 224, 224).cuda()
y = mresnet101(x)
y = (y**2).mean()
optim.zero_grad()
y.backward()
optim.step()