File: /home/panomity.de/sharp/src/sharp/models/composer.py
"""Defines module to compose final Gaussians from base values and delta values.
For licensing see accompanying LICENSE file.
Copyright (C) 2025 Apple Inc. All Rights Reserved.
"""
from __future__ import annotations
import torch
from torch import nn
from torch.nn import functional as F
from sharp.models.initializer import GaussianBaseValues
from sharp.utils import math as math_utils
from sharp.utils.color_space import ColorSpace, sRGB2linearRGB
from sharp.utils.gaussians import Gaussians3D
from .params import DeltaFactor
def _get_scale_activation_constant(max_scale: float, min_scale: float) -> tuple[float, float]:
"""Return constants for scale activation function."""
# To ensure for delta = 0, the value of scale_factor is 1 and the gradient is 1.
constant_a = (max_scale - min_scale) / (1 - min_scale) / (max_scale - 1)
constant_b = math_utils.inverse_sigmoid(
torch.tensor((1.0 - min_scale) / (max_scale - min_scale))
).item()
return constant_a, constant_b
class GaussianComposer(nn.Module):
"""Converts base values and deltas into Gaussians."""
color_activation_type: math_utils.ActivationType
opacity_activation_type: math_utils.ActivationType
def __init__(
self,
delta_factor: DeltaFactor,
min_scale: float,
max_scale: float,
color_activation_type: math_utils.ActivationType,
opacity_activation_type: math_utils.ActivationType,
color_space: ColorSpace,
base_scale_on_predicted_mean: bool,
scale_factor: int = 1,
) -> None:
"""Initialize GaussianComposer.
Args:
delta_factor: Multiply delta offsets by this factor.
min_scale: The minimal scale factor for gaussian scale activation.
max_scale: The maximal scale factor for gaussian scale activation.
color_activation_type: Which activation function to use for colors.
opacity_activation_type: Which activation function to use for opacities.
color_space: Which color space is used in training.
scale_factor: The scale factor to upsample the delta_values before composition.
base_scale_on_predicted_mean: Whether to account z offsets for estimating base scale.
"""
super().__init__()
self.delta_factor = delta_factor
self.max_scale = max_scale
self.min_scale = min_scale
self.color_activation_type = color_activation_type
self.opacity_activation_type = opacity_activation_type
self.color_space = color_space
self.scale_factor = scale_factor
self.base_scale_on_predicted_mean = base_scale_on_predicted_mean
def upsample_delta_value(self, delta: torch.Tensor, scale_factor: int = 1):
"""Upsample the delta value.
Args:
delta: The delta values predicted by gaussian predictor.
scale_factor: The scale factor to upsample the delta_values.
"""
(
batch_size,
num_channels,
num_layers,
image_height,
image_width,
) = delta.shape
new_height = image_height * scale_factor
new_width = image_width * scale_factor
upsampled_delta = F.interpolate(
delta.view(batch_size, num_channels * num_layers, image_height, image_width),
scale_factor=scale_factor,
).view(batch_size, num_channels, num_layers, new_height, new_width)
return upsampled_delta
def forward(
self,
delta: torch.Tensor,
base_values: GaussianBaseValues,
global_scale: torch.Tensor | None = None,
flatten_output: bool = True,
) -> Gaussians3D:
"""Combine predicted delta values with base gaussian values and apply activation function.
Args:
delta: The delta values predicted by gaussian predictor.
base_values: The gaussian base values.
global_scale: Global scale of Gaussians.
flatten_output: Flatten the gaussian parameters.
Returns:
The computed 3D Gaussians.
"""
# Upsample the delta if delta and base_values have different strides.
scale_factor = self.scale_factor
# For triplane head, the delta has already been upsampled.
actual_scale_factor = base_values.mean_x_ndc.shape[-1] // delta.shape[-1]
if scale_factor != 1 and actual_scale_factor != 1:
delta = self.upsample_delta_value(delta, scale_factor)
mean_vectors = self._forward_mean(base_values, delta)
# Account for the change in base scale due to z offsets.
base_scales = (
(base_values.scales * base_values.mean_inverse_z_ndc * mean_vectors[:, 2:3, ...])
if self.base_scale_on_predicted_mean
else base_values.scales
)
singular_values = self._scale_activation(
base_scales,
delta[:, 3:6],
self.min_scale,
self.max_scale,
)
quaternions = self._quaternion_activation(base_values.quaternions, delta[:, 6:10])
colors = self._color_activation(base_values.colors, delta[:, 10:13])
opacities = self._opacity_activation(base_values.opacities, delta[:, 13])
if flatten_output:
# [B, C, N, H, W] -> [B, N, H, W, C].
# NOTE: opacities is [B, N, H, W] so it doesn't need to permute.
mean_vectors = mean_vectors.permute(0, 2, 3, 4, 1).flatten(1, 3)
singular_values = singular_values.permute(0, 2, 3, 4, 1).flatten(1, 3)
quaternions = quaternions.permute(0, 2, 3, 4, 1).flatten(1, 3)
colors = colors.permute(0, 2, 3, 4, 1).flatten(1, 3)
opacities = opacities.flatten(1, 3)
# Apply global scaling to convert Gaussians to metric space.
if global_scale is not None:
mean_vectors = global_scale[:, None, None] * mean_vectors
singular_values = global_scale[:, None, None] * singular_values
return Gaussians3D(
mean_vectors=mean_vectors,
singular_values=singular_values,
quaternions=quaternions,
colors=colors,
opacities=opacities,
)
def _forward_mean(self, base_values: GaussianBaseValues, delta: torch.Tensor) -> torch.Tensor:
# Concatenate base vectors and apply mean activation.
delta_factor = torch.tensor(
[self.delta_factor.xy, self.delta_factor.xy, self.delta_factor.z],
device=delta.device,
)[None, :, None, None, None]
dtype = base_values.mean_x_ndc.dtype
device = base_values.mean_x_ndc.device
target_shape = (1, 3, 1, 1, 1)
mean_x_mask = torch.tensor([1.0, 0.0, 0.0], dtype=dtype, device=device).reshape(
target_shape
)
mean_y_mask = torch.tensor([0.0, 1.0, 0.0], dtype=dtype, device=device).reshape(
target_shape
)
mean_z_mask = torch.tensor([0.0, 0.0, 1.0], dtype=dtype, device=device).reshape(
target_shape
)
mean_vectors_ndc = (
base_values.mean_x_ndc.repeat(target_shape) * mean_x_mask
+ base_values.mean_y_ndc.repeat(target_shape) * mean_y_mask
+ base_values.mean_inverse_z_ndc.repeat(target_shape) * mean_z_mask
)
mean_vectors = self._mean_activation(mean_vectors_ndc, delta_factor * delta[:, :3])
return mean_vectors
def _mean_activation(self, base: torch.Tensor, learned_delta: torch.Tensor) -> torch.Tensor:
"""Mean activation function.
Args:
base: Tensor of shape [B, 3, H, W], where first two feature dimensions
(x,y) are in normalized device coordinates (NDC) where (-1, -1) is
the top, while the third dimension is inverse depth.
learned_delta: Tensor of shape [B, 3, H, W] with predicted delta values.
Returns:
Returns: The final mean vector after combining base and delta and applying nonlinearies.
"""
xx = base[:, 0:1] + learned_delta[:, 0:1]
yy = base[:, 1:2] + learned_delta[:, 1:2]
a = base[:, 2:3]
b = learned_delta[:, 2:3]
# Original formula:
inverse_zz = F.softplus(math_utils.inverse_softplus(a) + b)
zz = 1.0 / (inverse_zz + 1e-3)
mean_vectors = torch.cat([zz * xx, zz * yy, zz], dim=1)
return mean_vectors
def _scale_activation(
self,
base: torch.Tensor,
learned_delta: torch.Tensor,
min_scale: float,
max_scale: float,
) -> torch.Tensor:
constant_a, constant_b = _get_scale_activation_constant(max_scale, min_scale)
scale_factor = (max_scale - min_scale) * torch.sigmoid(
constant_a * self.delta_factor.scale * learned_delta + constant_b
) + min_scale
return base * scale_factor
def _quaternion_activation(
self, base: torch.Tensor, learned_delta: torch.Tensor
) -> torch.Tensor:
# No need to normalize the quaternions, since this is also done in rendering.
return base + self.delta_factor.quaternion * learned_delta
def _color_activation(self, base: torch.Tensor, learned_delta: torch.Tensor) -> torch.Tensor:
# For certain activation functions we need to clamp the base value to
# a supported range.
if self.color_activation_type == "sigmoid":
base = torch.clamp(base, min=0.01, max=0.99)
elif self.color_activation_type in ("exp", "softplus"):
base = torch.clamp(base, min=0.01)
activation = math_utils.create_activation_pair(self.color_activation_type)
colors: torch.Tensor = activation.forward(
activation.inverse(base) + self.delta_factor.color * learned_delta
)
# Convert gaussian color to linear if linearRGB colorspace is specified.
if self.color_space == "linearRGB":
colors = sRGB2linearRGB(colors)
return colors
def _opacity_activation(self, base: torch.Tensor, learned_delta: torch.Tensor) -> torch.Tensor:
activation = math_utils.create_activation_pair(self.opacity_activation_type)
return activation.forward(
activation.inverse(base) + self.delta_factor.opacity * learned_delta
)