import matplotlib.pyplot as plt
import pytorch3d
import torch
import imageio
import numpy as np

from starter.utils import get_device, get_mesh_renderer, load_cow_mesh
from pytorch3d.vis.plotly_vis import plot_scene
from solution import render_cow_360

1.1. 360-degree Renders

meshes, many_cameras, images = render_cow_360(
    cow_path="./data/cow.obj", image_size=256, color=[0.7, 0.7, 1], device=None,
)
print(images.shape)

torch.Size([60, 256, 256, 4])

plot_scene({
        "All Views": {
            "Mesh": meshes,
            "Cameras": many_cameras,
        },
    })

1.2 Re-creating the Dolly Zoom (10 points)

2.1 Constructing a Tetrahedron (5 points)

from solution2 import tetrahedron
vertices, faces, meshes, many_cameras, images = tetrahedron()
plot_scene({
        "All Views": {
            "Mesh": meshes,
            "Cameras": many_cameras,
        },
    })

print(f"Vertices has shape: {vertices.shape}")
print(f"Faces has shape: {faces.shape}")

Vertices has shape: torch.Size([1, 4, 3])
Faces has shape: torch.Size([1, 4, 3])

2.2 Constructing a Cube (5 points)

from solution2 import cube
vertices, faces, meshes, many_cameras, images = cube()
plot_scene({
        "All Views": {
            "Mesh": meshes,
            "Cameras": many_cameras,
        },
    })

print(f"Vertices has shape: {vertices.shape}")
print(f"Faces has shape: {faces.shape}")

Vertices has shape: torch.Size([1, 8, 3])
Faces has shape: torch.Size([1, 12, 3])

3. Re-texturing a mesh (10 points)

color1 = [1, 0, 1]
color2 = [1, 1, 2]
from solution2 import color_cow
meshes, many_cameras, images = color_cow(color1=color1, color2=color2)
plot_scene({
    "All Views": {
        "Mesh": meshes,
        "Cameras": many_cameras,
    },
})

print(f"With color1 {color1} and color2 {color2}, the color went from pink to white:")

With color1 [1, 0, 1] and color2 [1, 1, 2], the color went from pink to white:

4. Camera Transformations (20 points)

What R_relative is doing: After rotating (world coord) by R and translate by T, it will further rotate it by R_relative.

What T_relative is doing: Then it will translate the coord by T_relative.

from starter.camera_transforms import render_textured_cow

plt.imshow(render_textured_cow())

<matplotlib.image.AxesImage at 0x7fa4260dbdf0>

R_relative=[[0, -1., 0], [-1, 0, 0], [0, 0, 1]]
T_relative=[0, 0, 0]
print(f"We can rotate the camera by z-axis with -π/2 degrees with R_relative={R_relative} and T_relative={T_relative}")
plt.imshow(render_textured_cow(R_relative=R_relative, T_relative=T_relative))

We can rotate the camera by z-axis with -π/2 degrees with R_relative=[[0, -1.0, 0], [-1, 0, 0], [0, 0, 1]] and T_relative=[0, 0, 0]

/home/zhiqiul/.conda/envs/pytorch3d/lib/python3.9/site-packages/pytorch3d/transforms/transform3d.py:780: UserWarning:

R is not a valid rotation matrix

<matplotlib.image.AxesImage at 0x7fa4260d26a0>

# R_relative=[[0, -1., 0], [-1, 0, 0], [0, 0, 1]]
T_relative=[0, 0, 3]
print(f"We can move further away from the cow by going in -z direction with R_relative={R_relative} and T_relative={T_relative}")
print(f"Here the T_relative is the origin of world in camera coordinate.")
plt.imshow(render_textured_cow(T_relative=T_relative))

We can move further away from the cow by going in -z direction with R_relative=[[0, -1.0, 0], [-1, 0, 0], [0, 0, 1]] and T_relative=[0, 0, 3]
Here the T_relative is the origin of world in camera coordinate.

<matplotlib.image.AxesImage at 0x7fa420415340>

delta = 30 / 360.
R_relative=[[np.cos(delta), 0, np.sin(delta)], [0, 1, 0], [-np.sin(delta), 0, np.cos(delta)]]
T_relative=[0.3, -0.3, 0.]
print(f"We can rotate the camera by y-axis via {delta} radian with R_relative={R_relative}")
print(f"And then we move along the new + x direction and +y direction to get a slightly tilted view with T_relative={T_relative}")
plt.imshow(render_textured_cow(R_relative=R_relative, T_relative=T_relative))

We can rotate the camera by y-axis via 0.08333333333333333 radian with R_relative=[[0.9965297867005595, 0, 0.08323691620031025], [0, 1, 0], [-0.08323691620031025, 0, 0.9965297867005595]]
And then we move along the new + x direction and +y direction to get a slightly tilted view with T_relative=[0.3, -0.3, 0.0]

<matplotlib.image.AxesImage at 0x7fa42605da30>

delta = 180 / 360.
R_relative=[[0, 0, 1.0], [0, 1, 0], [-1, 0, 0]]
T_relative=[-3., 0, 3.]
print(f"We can rotate the camera by y-axis with π/2 with R_relative={R_relative}")
print(f"And then we move along the new +x direction and -z direction to get a recenter at cow with T_relative={T_relative}")
plt.imshow(render_textured_cow(R_relative=R_relative, T_relative=T_relative))

We can rotate the camera by y-axis with π/2 with R_relative=[[0, 0, 1.0], [0, 1, 0], [-1, 0, 0]]
And then we move along the new +x direction and -z direction to get a recenter at cow with T_relative=[-3.0, 0, 3.0]

<matplotlib.image.AxesImage at 0x7fa42036d7c0>

5.1 Rendering Point Clouds from RGB-D Images (10 points)

5.2 Parametric Functions (10 points)

5.3 Implicit Surfaces (15 points)

Rendering mesh vs point cloud:

• It is faster to render point cloud since it is a simple projection, whereas to render it as a mesh is substantially slower (not counting marching cube), because it involves rasterization and shading process.
• Mesh rendering is continuous whereas point cloud may have discontinuity if the sampled points are not enough.
• Point cloud could take more memory to achieve desired resolution. Whereas implicit function does not take that much memory. Meshes can also be optimized to reduce the storage requirement.

6. Do Something Fun (10 points)

Late day