class AdjacentCausalInjection(nn.Module):
"""BF16 adjacent injection and FP32 last-write refinement."""
def __init__(
self,
*,
hidden_size: int,
transfer_dim: int,
message_dim: int,
num_layers: int,
block_size: int,
params_dtype: torch.dtype,
quant_config: QuantizationConfig | None,
eps: float,
prefix: str,
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.transfer_dim = transfer_dim
self.num_layers = num_layers
self.block_size = block_size
self.eps = eps
def linear(input_size, output_size, name):
return ReplicatedLinear(
input_size,
output_size,
bias=False,
params_dtype=params_dtype,
quant_config=quant_config,
prefix=maybe_prefix(prefix, name),
return_bias=False,
)
self.readouts = nn.ModuleList(
linear(hidden_size, transfer_dim, f"readouts.{layer_idx}")
for layer_idx in range(num_layers)
)
self.writes = nn.ModuleList(
linear(message_dim, hidden_size, f"writes.{layer_idx}")
for layer_idx in range(num_layers)
)
self.message = linear(transfer_dim, message_dim, "message")
self.receiver_gate = linear(transfer_dim, message_dim, "receiver_gate")
self.predecessor_gate = linear(transfer_dim, message_dim, "predecessor_gate")
self.gate_bias = nn.Parameter(
torch.empty(message_dim, dtype=params_dtype), requires_grad=False
)
@staticmethod
def rms(states: torch.Tensor) -> torch.Tensor:
return (
states.float()
.square()
.mean(-1, keepdim=True)
.clamp_min(torch.finfo(torch.float32).tiny)
.sqrt()
)
def project(self, layer_idx: int, hidden_states: torch.Tensor) -> torch.Tensor:
normalized = hidden_states.float() / (self.rms(hidden_states) + self.eps)
projected = self.readouts[layer_idx](normalized.to(hidden_states.dtype)).float()
features = (
math.sqrt(self.transfer_dim)
* projected
/ (projected.norm(dim=-1, keepdim=True) + self.eps)
)
return features.to(hidden_states.dtype)
def token_features(
self, codebook: torch.Tensor, token_ids: torch.Tensor
) -> torch.Tensor:
return (codebook[token_ids.long()].float() * math.sqrt(self.transfer_dim)).to(
codebook.dtype
)
def write(
self,
layer_idx,
receiver_states,
receiver_features,
predecessor_features,
*,
refine: bool = False,
):
def linear(layer, features):
if refine:
return F.linear(features, layer.weight.float())
return layer(features).float()
gate = torch.sigmoid(
self.receiver_gate(receiver_features).float()
+ linear(self.predecessor_gate, predecessor_features)
+ self.gate_bias.float()
)
message = linear(self.message, predecessor_features)
gated_message = gate * message
if not refine:
gated_message = gated_message.to(receiver_states.dtype)
update = linear(self.writes[layer_idx], gated_message)
result = receiver_states.float() + self.rms(receiver_states) * update
return result if refine else result.to(receiver_states.dtype)
def forward(
self,
layer_idx: int,
pre_injection_states: torch.Tensor,
*,
anchor_features: torch.Tensor,
) -> torch.Tensor:
blocks = pre_injection_states.reshape(-1, self.block_size, self.hidden_size)
features = self.project(layer_idx, blocks)
predecessor = torch.cat((anchor_features[:, None, :], features[:, 1:-1]), dim=1)
successors = self.write(layer_idx, blocks[:, 1:], features[:, 1:], predecessor)
return torch.cat((blocks[:, :1], successors), dim=1).reshape_as(
pre_injection_states
)