base.py
定义BaseDataset Class(torch.utils.data.Dataset, ABC)
值得注意的 1. ABC作为parent class建立BaseDataset
分别有@abstractmethod 和 @staticmethod
abstractmethod 相当于java 中interface
只需要pass 而后subclass中具体根据用途定义
但是ABC class更强大在于可以同时有staticmethod 是实实在在的implemented method
@abstractmethod
def inner_get_item(self, idx):
pass
def __getitem__(self, idx):
try:
return self.inner_get_item(idx)
except Exception as e:
print(f"[DEBUG-DATASET] Error when loading {self.uids[idx]}")
# return self.__getitem__(idx+1)
raise e这里用了一个组合 使用内置indexing 会直接调用该subclass 的inner_get_item
使得每个sub class只需要重新写inner_get_item
- Static method部分
self.uids = json.load() 为dataset长度
_load_rgba_image converts Image.open()→np.array()→torch.form_numpy
rgba转换成rgb
rgba[:, :3, :, :] * rgba[:, 3:4, :, :]
rgb channel分别与 a(alpha 代表opacity透明度) channel相乘
之后**bg_color * (1 - rgba[:, 3:, :, :])** Computes the background color contribution where alpha is not fully opaque
如果本身全透明 (a channel为0) 则back_ground color 全部显现
若全不透明 (a channel为1) 则back_ground color 完全看不见
background与image作concat
还有一个查找dir的_locate_datadir
cam_utils.py
An extrinsic matrix represents the pose (position and orientation) of a camera or an object in a 3D space relative to a world coordinate system. It typically combines a rotation(缩放,旋转,shear剪切,reflect轴对称…) matrix and a translation vector(平移) to transform points from the world coordinate system to the camera coordinate system
总结一下: R是(3,3) T是(3,) RT是(3,4) Extrinsic Matrix是(4,4)
def compose_extrinsic_R_T(R: torch.Tensor, T: torch.Tensor):
"""
Compose the standard form extrinsic matrix from R and T.
Batched I/O.
"""
# (B,3,3) + (B,3,1) 末端加上translation matrix -> (B,3,4)
RT = torch.cat((R, T.unsqueeze(-1)), dim=-1)
return compose_extrinsic_RT(RT)def compose_extrinsic_RT(RT: torch.Tensor):
"""
Compose the standard form extrinsic matrix from RT.
Batched I/O.
"""
#右下角默认为1 -> (B,4,4) 这里先建最后一个row 然后repeat使前面dim对其
#batch dim后的dim 通过broadcast可以相加
return torch.cat([
RT,
torch.tensor([[[0, 0, 0, 1]]], dtype=RT.dtype, device=RT.device).repeat(RT.shape[0], 1, 1)
], dim=1)
后面两个decompose很简单就不贴了
normalize camera extrinsic matrices (poses) by applying a transformation that centers the cameras based on their distance to a pivotal point (often the center of the scene or object of interest)
def camera_normalization_objaverse(normed_dist_to_center, poses: torch.Tensor, ret_transform: bool = False):
assert normed_dist_to_center is not None
pivotal_pose = compose_extrinsic_RT(poses[:1])
#得到translation vector的normailized 值 这里取第一个pose做整个Dataset的reference
#因此b=1 用item取唯一值
dist_to_center = pivotal_pose[:, :3, 3].norm(dim=-1, keepdim=True).item() \
if normed_dist_to_center == 'auto' else normed_dist_to_center
# compute camera norm (new version)
canonical_camera_extrinsics = torch.tensor([[
[1, 0, 0, 0],
[0, 0, -1, -dist_to_center],
[0, 1, 0, 0],
[0, 0, 0, 1],
]], dtype=torch.float32)
pivotal_pose_inv = torch.inverse(pivotal_pose)
camera_norm_matrix = torch.bmm(canonical_camera_extrinsics, pivotal_pose_inv)
# normalize all views
poses = compose_extrinsic_RT(poses)
poses = torch.bmm(camera_norm_matrix.repeat(poses.shape[0], 1, 1), poses)
poses = decompose_extrinsic_RT(poses)
if ret_transform:
return poses, camera_norm_matrix.squeeze(dim=0)
return poses这里了解了另外一种一般calibration方法
Intrinsics * Extrinsics 构建calibration矩阵
将world point映射为image point 3d→2d 点
https://www.youtube.com/watch?v=qByYk6JggQU&ab_channel=FirstPrinciplesofComputerVision
https://www.youtube.com/watch?v=H5qbRTikxI4&t=704s&ab_channel=KevinWood%7CRobotics%26AI
这里是将所有所有poses建立在canonical frame之上
camera alignment
Inverse matrix reverses linear transformation
因为有一段时间没有接触线性代数了 这里写的很慢 比较重复
可以想象从pos1 开始 相机会一直变换角度
我们相当于把每个pose除以最开始的pos(做reverse transformation)
得到转变量 之后再transform到canonical frame上
这样pos0因为 自身就是起点 转变量就是I 所以最终pose就是frame量本身
linear transformation → 看align上去的值
reverse transformation → 看mis align的值 转变量
Normalize 一下Intrinsics 内参
def get_normalized_camera_intrinsics(intrinsics: torch.Tensor):
"""
intrinsics: (N, 3, 2), [[fx, fy], [cx, cy], [width, height]]
Return batched fx, fy, cx, cy
"""
fx, fy = intrinsics[:, 0, 0], intrinsics[:, 0, 1]
cx, cy = intrinsics[:, 1, 0], intrinsics[:, 1, 1]
width, height = intrinsics[:, 2, 0], intrinsics[:, 2, 1]
fx, fy = fx / width, fy / height
cx, cy = cx / width, cy / height
return fx, fy, cx, cy
def build_camera_principle(RT: torch.Tensor, intrinsics: torch.Tensor):
"""
RT: (N, 3, 4)
intrinsics: (N, 3, 2), [[fx, fy], [cx, cy], [width, height]]
"""
fx, fy, cx, cy = get_normalized_camera_intrinsics(intrinsics)
return torch.cat([
RT.reshape(-1, 12),
fx.unsqueeze(-1), fy.unsqueeze(-1), cx.unsqueeze(-1), cy.unsqueeze(-1),
], dim=-1)这里的broadcast 将(b,12) 与 4个 (1,1) concat在一起
[(b,12),(1，1)…] 第一个dim broadcast到dim上
因为camera不变 所有RT poses concat相同量
看错了 从上一步来的fx, fy 都是(b,)
这里单纯前面batch dim一致 在dim=-1 多加4个数
torch.cat不能broadcast 只有加减乘除可以
有点跑题了 试了试concat和repeat的机制 回到下一个function
def build_camera_standard(RT: torch.Tensor, intrinsics: torch.Tensor):
"""
RT: (N, 3, 4)
intrinsics: (N, 3, 2), [[fx, fy], [cx, cy], [width, height]]
"""
E = compose_extrinsic_RT(RT)
fx, fy, cx, cy = get_normalized_camera_intrinsics(intrinsics)
I = torch.stack([
torch.stack([fx, torch.zeros_like(fx), cx], dim=-1),
torch.stack([torch.zeros_like(fy), fy, cy], dim=-1),
torch.tensor([[0, 0, 1]], dtype=torch.float32, device=RT.device).repeat(RT.shape[0], 1),
], dim=1)
return torch.cat([
E.reshape(-1, 16),
I.reshape(-1, 9),
], dim=-1)
像之前提到的构建完整Intrinsic matrix(I)
和之前function差不多 只是现在包含更规范 dim
构建完整matrix用stack更合适 如果只是一个dim上合并 concat适合
def center_looking_at_camera_pose(
camera_position: torch.Tensor, look_at: torch.Tensor = None, up_world: torch.Tensor = None,
device: torch.device = torch.device('cpu'),
):
"""
camera_position: (M, 3)
look_at: (3)
up_world: (3)
return: (M, 3, 4)
"""
# by default, looking at the origin and world up is pos-z
if look_at is None:
look_at = torch.tensor([0, 0, 0], dtype=torch.float32, device=device)
if up_world is None:
up_world = torch.tensor([0, 0, 1], dtype=torch.float32, device=device)
look_at = look_at.unsqueeze(0).repeat(camera_position.shape[0], 1)
up_world = up_world.unsqueeze(0).repeat(camera_position.shape[0], 1)
z_axis = camera_position - look_at
z_axis = z_axis / z_axis.norm(dim=-1, keepdim=True)
x_axis = torch.cross(up_world, z_axis)
x_axis = x_axis / x_axis.norm(dim=-1, keepdim=True)
y_axis = torch.cross(z_axis, x_axis)
y_axis = y_axis / y_axis.norm(dim=-1, keepdim=True)
extrinsics = torch.stack([x_axis, y_axis, z_axis, camera_position], dim=-1)
return extrinsics定义look_at 和 up_world(0,0,1)(default z axis up)
z_axis = camera_position - look_at
normalize(individually through layer norm)
def surrounding_views_linspace(n_views: int, radius: float = 2.0, height: float = 0.8, device: torch.device = torch.device('cpu')):
"""
n_views: number of surrounding views
radius: camera dist to center
height: height of the camera
return: (M, 3, 4)
"""
assert n_views > 0
assert radius > 0
theta = torch.linspace(-torch.pi / 2, 3 * torch.pi / 2, n_views, device=device)
projected_radius = math.sqrt(radius ** 2 - height ** 2)
x = torch.cos(theta) * projected_radius
y = torch.sin(theta) * projected_radius
z = torch.full((n_views,), height, device=device)
camera_positions = torch.stack([x, y, z], dim=1)
extrinsics = center_looking_at_camera_pose(camera_positions, device=device)
return extrinsics