>>102081629
It's from the ground up, initialized from the void. Basically the Pixart Sigma except with 1.3B parameters and I'm using Adam Mini as the optimizer to train it and it's better than 8 bit Adam because it's stays in FP16. (Flux could probably be trained the same since it's a transformers model)
>>102081646
@MODELS.register_module()
def PixArtMS_XL_2_Next(**kwargs):
return PixArtMS(
depth=42,
hidden_size=1408,
patch_size=2,
num_heads=16,
**kwargs
)
kv_compress = True
kv_compress_config = {
'sampling': 'conv', # ['conv', 'uniform', 'ave']
'scale_factor': 2,
'kv_compress_layer': [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41],
}
