CMU10-414/714 Deep Learning Systems 第二次作业hw1解析

def gradient(self, out_grad, node):
    if self.axes:
        x, y = self.axes[0], self.axes[1]
    else:
        # 如果 axes 是 None，默认交换最后两个轴
        x, y = -1, -2
    return transpose(out_grad, axes=(x, y))

def gradient(self, out_grad, node):
    ### BEGIN YOUR SOLUTION
    # 将梯度 reshape 成原来的形状
    return reshape(out_grad, node.inputs[0].shape)
    ### END YOUR SOLUTION

def gradient(self, out_grad, node):
    lhs_shape = node.inputs[0].shape  # 原始输入的形状
    output_shape = self.shape  # 广播后的目标形状

# 如果输入形状和输出形状相同，说明没有发生广播，直接返回 out_grad
if lhs_shape == output_shape:
return out_grad

# 计算输入和输出形状的维度差，处理输入比输出少的维度情况
    ndim_diff =len(output_shape)-len(lhs_shape)

# 需要缩减的维度
    reduce_axes =[]

# 遍历输出张量的每个维度，找到需要缩减的维度
for i inrange(len(output_shape)):
# 计算对应的输入维度，如果是补上去的（即左边加 1），则视为 1
        input_dim = lhs_shape[i - ndim_diff]if i >= ndim_diff else1
        output_dim = output_shape[i]

# 如果输入维度为 1，而输出维度大于 1，则需要在该轴进行求和
if input_dim ==1and output_dim !=1:
            reduce_axes.append(i)

# 如果 reduce_axes 非空，执行 sum 操作以消除广播的影响
if reduce_axes:
# out_grad = array_api.sum(out_grad.numpy(), axis=tuple(reduce_axes))
        out_grad = summation(out_grad, axes=tuple(reduce_axes))

# reshape 回原始输入的形状
return reshape(out_grad, lhs_shape)

def gradient(self, out_grad, node):
    lhs = node.inputs[0]
    input_shape = lhs.shape  # 原始张量的形状

if out_grad.shape ==():
        out_grad = array_api.full_like(lhs.numpy(), out_grad.numpy())

if self.axes isNone:
# 如果没有指定轴，意味着我们对整个张量求和，结果是一个标量
return broadcast_to(ndl.Tensor(out_grad), input_shape)

    out_grad = out_grad.numpy()ifisinstance(out_grad,Tensor)else out_grad

# 为 out_grad 恢复被求和的维度
if out_grad.shape != input_shape:
for axis insorted(self.axes, reverse=True):
          out_grad = array_api.expand_dims(out_grad, axis)

# 扩展为原始输入的形状
    result = broadcast_to(ndl.Tensor(out_grad), input_shape)
return result

def gradient(self, out_grad, node):
    lhs, rhs = node.inputs

# 始终使用批量矩阵乘法的逻辑
    grad_lhs = matmul(out_grad, transpose(rhs, axes=(-2,-1)))
    grad_rhs = matmul(transpose(lhs, axes=(-2,-1)), out_grad)

# 处理广播的维度
iflen(lhs.shape)<len(rhs.shape):
        axes_to_sum =tuple(range(len(rhs.shape)-len(lhs.shape)))
        grad_lhs = summation(grad_lhs, axes=axes_to_sum)
iflen(rhs.shape)<len(lhs.shape):
        axes_to_sum =tuple(range(len(lhs.shape)-len(rhs.shape)))
        grad_rhs = summation(grad_rhs, axes=axes_to_sum)

return grad_lhs, grad_rhs

def gradient(self, out_grad, node):
    lhs = node.inputs[0]
    np_lhs = lhs.numpy()
    # 创建一个掩码：lhs > 0 的位置为 1，其他位置为 0
    grad_mask = array_api.where(np_lhs > 0, 1, 0)
    # 将 out_grad 和 grad_mask 相乘，得到最终的梯度
    return out_grad * ndl.Tensor(grad_mask)

    A
   / \
  B   C
 / \   \
D   E   F

def find_topo_sort(node_list: List[Value])->List[Value]:
    visited =set()# 用于记录已访问过的节点
    topo_order =[]# 存储拓扑排序的结果

# 遍历节点列表，执行 DFS
for node in node_list:
if node notin visited:
            topo_sort_dfs(node, visited, topo_order)

# 返回拓扑排序的结果，由于我们在 DFS 中是后序添加节点的，因此需要反转
return topo_order

deftopo_sort_dfs(node, visited, topo_order):
"""Post-order DFS"""
# 如果当前节点已经访问过，直接返回
if node in visited:
return
    visited.add(node)# 标记当前节点为已访问

# 遍历所有前驱节点（即指向当前节点的节点），递归执行DFS
for predecessor in node.inputs:
if predecessor notin visited:
            topo_sort_dfs(predecessor, visited, topo_order)

# 在所有前驱节点都被访问后，将当前节点加入到拓扑排序的结果中
    topo_order.append(node)

def compute_gradient_of_variables(output_tensor, out_grad):
"""Take gradient of output node with respect to each node in node_list.

    Store the computed result in the grad field of each Variable.
    """
# Step 1: Dictionary that records a list of partial adjoints of each node
    node_to_grad ={output_tensor:[out_grad]}

# Step 2: Traverse nodes in reverse topological order (i.e., from output to input)
    reverse_topo_order =list(reversed(find_topo_sort([output_tensor])))

for node in reverse_topo_order:
# Step 3: Sum up partial adjoints        
        v_i = sum_node_list(node_to_grad[node])
# Step 4: Store the computed gradient in the node's grad field
        node.grad = v_i
if node.is_leaf():
continue
        gradient = node.op.gradient_as_tuple(v_i, node)
# Step 5: Propagate gradients to input nodes
for i, node1 inenumerate(node.inputs):
ifnot node_to_grad.get(node1):
              node_to_grad[node1]=[]
            node_to_grad[node1].append(gradient[i])

def softmax_loss(Z, y_one_hot):
    exp_sum_z = ndl.summation(ndl.exp(Z), axes=(-1,))
    b = Z.shape[0]
    z_y = ndl.summation(ndl.multiply(Z, y_one_hot), axes=(-1,))
    loss = ndl.summation(ndl.log(exp_sum_z) - z_y) / b
    return loss

def nn_epoch(X, y, W1, W2, lr=0.1, batch=100):
    num_examples, input_dim = X.shape
    _, num_classes = W2.shape
for i inrange(0, num_examples, batch):
        X_batch = X[i:i + batch]
        y_batch = y[i:i + batch]
        x = ndl.Tensor(X_batch)

        y_one_hot = np.zeros((y_batch.size, num_classes))
        y_one_hot[np.arange(y_batch.size), y_batch]=1
        y_tensor = ndl.Tensor(y_one_hot)

        Z = ndl.matmul(ndl.relu(ndl.matmul(x, W1)), W2)
        loss = softmax_loss(Z, y_tensor)
        loss.backward()

        W1 = ndl.Tensor(W1.numpy()- lr * W1.grad.numpy())
        W2 = ndl.Tensor(W2.numpy()- lr * W2.grad.numpy())
return W1, W2