README.md

@@ -25,16 +25,67 @@ git checkout solutions
 git merge master
 ```
 
-You can create conda environment with all required packages via 
+You can create conda environment with all required packages via  the following for CPU:
 
 ```bash
-cd conda_env_yaml
-# For CPU-only setup run
-conda env create -f environment-cpu.yml
-# if you have CUDA GPU card use instead
-conda env create -f environment-gpu.yml
+conda create --name mpv-assignments-cpu-only python=3.10
+conda activate mpv-assignments-cpu-only
+pip3 install torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cpu
+pip3 install kornia==0.6.10 tqdm notebook matplotlib opencv-contrib-python==4.7.0.68 seaborn tensorboard tensorboardX ipywidgets widgetsnbextension
+pip3 install kornia_moons --no-deps
 ```
 
+And following for GPU. You may need to change the cuda version to the actually installed one.
+For the GPU setup, if you have CUDA-capable GPU (if needed - change CUDA version in command). To find out your CUDA version, run `nvidia-smi`. You will see something like:
 
+```
+Mon Feb 20 16:49:46 2023       
++-----------------------------------------------------------------------------+
+| NVIDIA-SMI 460.39       Driver Version: 460.39       CUDA Version: 11.2     |
+|-------------------------------+----------------------+----------------------+
+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+|                               |                      |               MIG M. |
+|===============================+======================+======================|
+|   0  GeForce GTX 108...  On   | 00000000:01:00.0 Off |                  N/A |
+| 25%   44C    P8    18W / 250W |      1MiB / 11178MiB |      0%      Default |
+|                               |                      |                  N/A |
++-------------------------------+----------------------+----------------------+
+|   1  GeForce GTX 108...  On   | 00000000:06:00.0 Off |                  N/A |
+|ERR!   54C    P0   ERR! / 250W |      1MiB / 11178MiB |     75%      Default |
+|                               |                      |                  N/A |
++-------------------------------+----------------------+----------------------+
+                                                                               
++-----------------------------------------------------------------------------+
+| Processes:                                                                  |
+|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
+|        ID   ID                                                   Usage      |
+|=============================================================================|
+|  No running processes found                                                 |
++-----------------------------------------------------------------------------+
+```
+In the example above, the CUDA version is 11.2, so you should use `–extra-index-url https://download.pytorch.org/whl/cu112`
+
+
+
+```bash
+conda create --name mpv-assignments-gpu python=3.10
+conda activate mpv-assignments-gpu
+pip3 install torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu112
+pip3 install kornia==0.6.10 tqdm notebook matplotlib opencv-contrib-python==4.7.0.68 seaborn tensorboard tensorboardX ipywidgets widgetsnbextension
+pip3 install kornia_moons --no-deps
+```
+
+For Apple Silicon devices (M1, M2 family) use:
+
+```bash
+conda create --name mpv-assignments-cpu-only python=3.10
+conda activate mpv-assignments-cpu-only
+conda install -c apple tensorflow-deps
+pip3 install tensorflow-macos tensorflow-metal
+pip3 install torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cpu
+pip3 install kornia==0.6.10 tqdm notebook matplotlib opencv-contrib-python==4.7.0.68 seaborn tensorboard tensorboardX ipywidgets widgetsnbextension
+pip3 install kornia_moons --no-deps
+```
 
 **Keep in mind that the assignments and the assignment templates will be updated during the semester.  Always pull the current template version before starting to work on an assignment!**

assignment_0_3_correspondences_template/RANSAC/plot_planar.py

+from copy import deepcopy
+
+import cv2
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from dataclasses import dataclass
+from matplotlib.patches import Patch
+import kornia.feature as KF
+from RANSAC.ransac_planar_params import RansacPlanarLog
+
+
+@dataclass
+class PlotParams:
+    cross_size_img_size_ratio = 80.                     # all keypoints are displayed as crosses
+    cross_thickness_img_size_ratio = 250.               # all keypoints are displayed as crosses
+    line_thickness_img_size_ratio = 300.                # correspondences are connected with this line
+    perspective_map_thickness_img_size_ratio = 200.     # homography is displayed as a polygon
+    perspective_line_thickness_img_size_ratio = 150     # homography is displayed as a polygon
+    inlier_color = (102., 255., 51.)                    # green - color of crosses and lines
+    outlier_color = (255., 26., 26.)                    # red - color of crosses and lines
+    kps_color = (179., 138., 77.)                       # dim orrange - color of keypoints
+    perspective_map_color = (0., 0., 255.)              # blue - color of the homography polygon
+    best_perspective_map_color = (255., 255., 255.)     # white - color of the best homography so far
+    position_based_color_left = True                    # if True left image keypoints are colored based on position
+    draw_lafs = True                                    # if True, the features are drawn as ellipces.
+
+
+
+    @staticmethod
+    def ratio_to_size(img, ratio):
+        """
+        We want to keep sizes relative to the image size.
+        This function defines how we obtain a value that we use in library visualization functions.
+        """
+        if len(img.shape) == 3:
+            return int(min(img.shape[:2]) / ratio)
+        return int(min(img.shape) / ratio)
+
+
+def draw_cross(img, center, color, thickness, d):
+    cv2.line(img, (center[0] - d, center[1] - d), (center[0] + d, center[1] + d), color, thickness, cv2.LINE_AA, 0)
+    cv2.line(img, (center[0] + d, center[1] - d), (center[0] - d, center[1] + d), color, thickness, cv2.LINE_AA, 0)
+    return img
+
+
+def add_offset(img_left, pts):
+    """
+    pts.shape = N, 4
+    img_left is the left image -> to get the offset size
+    """
+    if not isinstance(pts, torch.Tensor):
+        pts = torch.from_numpy(pts)
+    if pts is not None and len(pts) > 0:
+        return pts + torch.tensor([0, 0, img_left.shape[1], 0])
+    return pts
+
+
+def draw_crosses(img, pts: torch.Tensor, color, plot_params, colors = None):
+    """ pts.shape = N, 2 """
+    min_side = min(img.shape)
+    cross_size = PlotParams.ratio_to_size(img, plot_params.cross_size_img_size_ratio)
+    thickness = PlotParams.ratio_to_size(img, plot_params.cross_thickness_img_size_ratio)
+    if not isinstance(pts, torch.Tensor):
+        pts = torch.from_numpy(pts)
+    for i, pt in enumerate(pts.round().to(dtype=torch.int32)):
+        if colors is None:
+            current_color  = color
+        else:
+            current_color = colors[i]
+        img = draw_cross(img, pt.detach().cpu().numpy(), current_color, thickness, cross_size)
+    return img
+
+
+def draw_ellipse(img, contour, color, thickness):
+    img = cv2.polylines(img, contour, True, color, thickness)
+    return img
+
+
+def draw_ellipses(img, pts: torch.Tensor, color, plot_params, colors = None):
+    """ pts.shape = N, 2 """
+    min_side = min(img.shape)
+    cross_size = PlotParams.ratio_to_size(img, plot_params.cross_size_img_size_ratio)
+    thickness = PlotParams.ratio_to_size(img, plot_params.cross_thickness_img_size_ratio)
+    if not isinstance(pts, torch.Tensor):
+        pts = torch.from_numpy(pts)
+    boundary_pts = KF.laf_to_boundary_points(pts, 30).reshape(-1, 30, 2)
+
+    for i, contour in enumerate(boundary_pts):
+        if colors is None:
+            current_color  = color
+        else:
+            current_color = colors[i]
+        img = draw_ellipse(img, contour.unsqueeze(0).detach().cpu().numpy().astype(np.int32), current_color, thickness)
+    return img
+
+
+def draw_lines(img, pts: torch.Tensor, color, plot_params):
+    """ pts.shape = N, 4 """
+    thickness = PlotParams.ratio_to_size(img, plot_params.line_thickness_img_size_ratio)
+    for pt in pts.round().to(dtype=torch.int32):
+        pt = pt.detach().cpu().numpy()
+        img = cv2.line(img, tuple(pt[:2]), tuple(pt[2:]), color, thickness)
+    return img
+
+
+def join_images(img1, img2):
+    draw_params = dict(matchColor=(255, 255, 0),
+                       singlePointColor=None,
+                       matchesMask=[],
+                       flags=20)
+    return cv2.drawMatches(img1, [], img2, [], [], None, **draw_params)
+
+
+def draw_perspective_map(img1, img, H, color, plot_params: PlotParams):
+    if H is not None:
+        h, w, ch = img1.shape
+        pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
+        thickness = PlotParams.ratio_to_size(img, plot_params.perspective_line_thickness_img_size_ratio)
+        if isinstance(H, torch.Tensor):
+            H = H.detach().cpu().numpy()
+        dst = cv2.perspectiveTransform(pts, H)
+        res = cv2.polylines(img, [np.int32(dst)], True, color, thickness, cv2.LINE_AA)
+        return res
+    return img
+
+
+def decolorize(img):
+    """ de-colorizes the image and KEEPS the dimensions! """
+    assert len(img.shape) in {2, 3}, f"Image is not in correct shape (H, W, Optional[C]), shape: {img.shape}"
+    res = img.copy()
+    if len(img.shape) == 3:
+        res = cv2.cvtColor(res, cv2.COLOR_BGR2GRAY)
+    return cv2.cvtColor(res, cv2.COLOR_GRAY2RGB)
+
+
+def draw_keypoints(img1, img2, kps1, kps2, plt_params: PlotParams):
+    imga, imgb = decolorize(img1), decolorize(img2)  # RGB -> GRAY -> GRAY RGB
+    colors = None
+    if not plt_params.draw_lafs:
+        imga = draw_crosses(imga, KF.get_laf_center(kps1).view(-1, 2), plt_params.kps_color, plt_params, colors=colors)  # keypoints
+        imgb = draw_crosses(imgb, KF.get_laf_center(kps2).view(-1, 2), plt_params.kps_color, plt_params, colors=colors)  # keypoints
+    else:
+        imga = draw_ellipses(imga, kps1, plt_params.kps_color, plt_params, colors=colors)  # keypoints
+        imgb = draw_ellipses(imgb, kps2, plt_params.kps_color, plt_params, colors=colors)  # keypoints
+    imgab = join_images(imga, imgb)
+    return imgab
+
+
+def draw_matches(img1, img2, detections, plt_params: PlotParams,
+                 visualization_options: dict):
+    """
+    @param visualization_options: a dictionary that defines how to display tentative correspondences.
+                                  Modes works as follows: there are 3 modes 0, 1, 2, 3
+        #                         Mode 0: do not plot anything
+        #                         Mode 1: show only keypoints (in both images)
+        #                         Mode 2: show regions (in both images)
+        #                         Mode 3: show keypoints and connect them with lines
+        #                         Mode 4: show regions and connect them with lines
+    """
+    # todo: pts_matches none, mask None etc etc
+    imga, imgb = decolorize(img1), decolorize(img2)  # RGB -> GRAY -> GRAY RGB
+    #
+    #Detections = namedtuple('Detections', ['kps1',
+#                                       'kps2',
+    #                                   'tentative_matches',
+    #                                   'pts_matches',
+    #                                   'H',
+    #                                   'inlier_mask',
+    #                                   'lafs1',
+    #                                   'lafs2'])
+
+    idxs = detections.tentative_matches
+    tents_1 =  detections.lafs1[0:1,idxs[:, 0]]
+    tents_2 =  detections.lafs2[0:1,idxs[:, 1]]
+    inlier_mask = detections.inlier_mask.reshape(-1).bool()
+    assert len(inlier_mask) == tents_1.shape[1], f"pts_matches ({len(tents_1)}) and mask ({len(inlier_mask)}) lengths differ"
+    if len(inlier_mask) == 0:
+        return
+    if plt_params.position_based_color_left:
+        colors_out = []
+        for i, pt in enumerate(KF.get_laf_center(tents_1).view(-1, 2)):
+            current_color = (255 * (pt[1].item() /imga.shape[0]),
+                             0,
+                             255 * (pt[0].item() /imga.shape[1]))
+            colors_out.append(current_color)
+    else:
+        colors_out = None
+
+    # drawing outliers first
+
+    if visualization_options['outliers_display_mode'] in [1, 3]:
+        imga = draw_crosses(imga, KF.get_laf_center(tents_1).view(-1, 2)[~inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+        imgb = draw_crosses(imgb, KF.get_laf_center(tents_2).view(-1, 2)[~inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+    if visualization_options['outliers_display_mode'] in [2, 4]:
+        imga = draw_ellipses(imga, tents_1[:,~inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+        imgb = draw_ellipses(imgb, tents_2[:,~inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+    if visualization_options['inliers_display_mode'] in [1, 3]:
+        imga = draw_crosses(imga, KF.get_laf_center(tents_1).view(-1, 2)[inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+        imgb = draw_crosses(imgb, KF.get_laf_center(tents_2).view(-1, 2)[inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+    if visualization_options['inliers_display_mode'] in [2, 4]:
+        imga = draw_ellipses(imga, tents_1[:,inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+        imgb = draw_ellipses(imgb, tents_2[:,inlier_mask], plt_params.kps_color, plt_params, colors=colors_out)  # keypoints
+
+
+
+    if detections.H is not None:    # best so far perspective map
+        imgb = draw_perspective_map(imga, imgb, detections.H, plt_params.perspective_map_color, plt_params)  # perspective map
+    # ----------- imgab
+    pts_matches = torch.cat([KF.get_laf_center(tents_1).view(-1, 2),
+                             KF.get_laf_center(tents_2).view(-1, 2)], dim=1)
+    pts_offset = add_offset(img1, pts_matches)
+    imgab = join_images(imga, imgb)
+    if visualization_options['outliers_display_mode'] in [3, 4]:
+        imgab = draw_lines(imgab, pts_offset[~inlier_mask], plt_params.outlier_color, plt_params)  # outliers connections
+    if visualization_options['inliers_display_mode']  in [3, 4]:
+        imgab = draw_lines(imgab, pts_offset[inlier_mask], plt_params.inlier_color, plt_params)  # inliers connections
+    return imgab
+
+
+def plot_legend_init(kps1, kps2, plt_params, fontsize=13):
+    plt.rc('legend', fontsize=fontsize)
+    fig = plt.figure(figsize=(.01, .01))
+    ax = fig.add_subplot(111)
+    # legend_items = [Tuple[handle, label]] -> I do this to ensure the correct order
+    legend_items = [
+        (ax.scatter([], [], color=np.array(plt_params.kps_color)/255., marker='x'), f'keypoints1: {len(kps1)}'),
+        (ax.scatter([], [], color=np.array(plt_params.kps_color)/255., marker='x'), f'keypoints2: {len(kps2)}'),
+    ]
+    handles, labels = list(zip(*legend_items))
+    plt.axis('off')
+    ax.legend(handles, labels, loc='center')
+    plt.show()
+    return handles, labels
+
+
+def plot_best_legend(log, detections, plt_params, fontsize=13):
+    plt.rc('legend', fontsize=fontsize)
+    fig = plt.figure(figsize=(.01, .01))
+    ax = fig.add_subplot(111)
+    legend_items = [
+        (ax.plot([], [], ' ')[0], "$\\bf{best\ so\ far}$" + f" - i: {log.i + 1}"),
+        (ax.plot([], [], ' ')[0], f'inliers: {detections.inlier_mask.long().sum().item()}'),
+        (ax.plot([], [], ' ')[0], f'outliers: {len(detections.inlier_mask.reshape(-1)) - detections.inlier_mask.long().sum().item()}'),
+        (ax.plot([], [], ' ')[0], f'loss: {log.loss:.2f}'),
+        (Patch(facecolor=np.array(plt_params.best_perspective_map_color)/255., edgecolor='black'), 'perspective map'),
+    ]
+    handles, labels = list(zip(*legend_items))
+    plt.axis('off')
+    ax.legend(handles, labels, loc='center')
+    plt.show()
+    return handles, labels
+
+
+def plot_current_legend(log, detections, plt_params, fontsize=13):
+    plt.rc('legend', fontsize=fontsize)
+    fig = plt.figure(figsize=(.01, .01))
+    ax = fig.add_subplot(111)
+    # legend_items = [Tuple[handle, label]] -> I do this to ensure the correct order
+    legend_items = [
+        (ax.plot([], [], ' ')[0], "$\\bf{current}$" + f" - i: {log.i + 1}"),
+        (ax.scatter([], [], color=np.array(plt_params.inlier_color)/255., marker='x'),
+            f'inliers: {detections.inlier_mask.long().sum().item()}'),
+        (ax.scatter([], [], color=np.array(plt_params.outlier_color)/255., marker='x'),
+            f'outliers: {len(detections.inlier_mask.reshape(-1)) - detections.inlier_mask.long().sum().item()}'),
+        (ax.plot([], color=np.array(plt_params.inlier_color)/255.)[0], f'loss: {log.loss:.2f}'),
+        (Patch(facecolor=np.array(plt_params.perspective_map_color)/255., edgecolor='black'), 'perspective map'),
+    ]
+    handles, labels = list(zip(*legend_items))
+    plt.axis('off')
+    ax.legend(handles, labels, loc='center')
+    plt.show()
+    return handles, labels
+
+
+def cv2_draw_matches(kps1, kps2, tentative_matches, H, inlier_mask, img1, img2, H_gt = None):
+    """
+    detections = Detections(*detect_describe_match(timg1, timg2, params, functions))
+    log = ransac_planar(detections.pts_matches, params, functions)
+    imgout = cv2_draw_matches(detections.kps1, detections.kps2, detections.tentative_matches,
+             log.H, log.mask, decolorize(img1), decolorize(img2))
+    """
+    matches_mask = inlier_mask.ravel().tolist()
+    h, w, ch = img1.shape
+    pts = np.float32([[0, 0], [0, h-1], [w-1, h-1], [w-1, 0]]).reshape(-1, 1, 2)
+    dst = cv2.perspectiveTransform(pts, H)
+    img2_tr = cv2.polylines(decolorize(img2), [np.int32(dst)], True, (0, 0, 255), 3, cv2.LINE_AA)
+    if H_gt is not None:        # Ground truth transformation
+        dst_gt = cv2.perspectiveTransform(pts, H_gt)
+        img2_tr = cv2.polylines(deepcopy(img2_tr), [np.int32(dst_gt)], True, (0, 255, 0), 3, cv2.LINE_AA)
+    # Blue is estimated, green is ground truth homography
+    draw_params = dict(matchColor=(255, 255, 0), singlePointColor=None,
+                       matchesMask=matches_mask, flags=20)
+    img_out = cv2.drawMatches(decolorize(img1), kps1, img2_tr, kps2, tentative_matches, None, **draw_params)
+    plt.figure(figsize=(20, 10))
+    plt.imshow(img_out)
+    plt.show()
+    return img_out

assignment_0_3_correspondences_template/RANSAC/ransac_planar_params.py

+from typing import Callable
+
+import numpy as np
+from dataclasses import dataclass, field
+DBL_EPS = np.finfo(float).eps
+
+
+@dataclass
+class RansacPlanarLog:
+    """ future tip: make it comparable as used in update """
+    H: np.array = field(default=None, repr=False)
+    mask: np.array = field(default=None, repr=False)
+    n: int = 0
+    loss: float = np.inf
+    i: int = 0
+
+    def __post_init__(self):
+        if self.mask is not None:
+            self.n = self.mask.sum().item()
+
+    def __lt__(self, other):
+        if self.n != other.n:
+            return self.n < other.n
+        return self.loss-DBL_EPS > other.loss
+
+    def __gt__(self, other):
+        return other < self
+
+    def __eq__(self, other):
+        return not (self < other or other < self)
+
+
+@dataclass
+class RansacPlanarParams:
+    sample_size: int = 4
+    ransac_th: float = 15.0  # pixel threshold for correspondence to be counted as inlier
+    max_iter: int = 10000    # maximum iteration, overrides confidence
+    conf: float = 0.99       # confidence
+
+
+@dataclass(eq=False)
+class RansacPlanarFunctions:
+    nsamples: Callable = None
+    getH: Callable = None                   # estimates homography from minimal sample
+    hdist: Callable = None                  # calculates one - way reprojection error
+    sampler: Callable = None                # draws random sample from pts_matches
+    check_sample: Callable = None           # is a set of samples valid for homography estimation?

assignment_0_3_correspondences_template/imagefiltering.py

@@ -41,7 +41,9 @@ def filter2d(x: torch.Tensor, kernel: torch.Tensor) -> torch.Tensor:
         as the input.
     """
     out =  x
-    out =  x
+    ## Do not forget about flipping the kernel!
+    ## See in details here https://towardsdatascience.com/convolution-vs-correlation-af868b6b4fb5
+    
     return out
 
 def gaussian_filter2d(x: torch.Tensor, sigma: float) -> torch.Tensor:
@@ -59,7 +61,7 @@ def gaussian_filter2d(x: torch.Tensor, sigma: float) -> torch.Tensor:
 
     """ 
     ksize = get_gausskernel_size(sigma)
-    out =  x
+    out = x
     return out
 
 
@@ -80,22 +82,6 @@ def spatial_gradient_first_order(x: torch.Tensor, sigma: float) -> torch.Tensor:
     return out
 
 
-def spatial_gradient_second_order(x: torch.Tensor, sigma: float) -> torch.Tensor:
-    r"""Computes the second order image derivative in xx, xy, yy directions using Gaussian derivative
-
-    Return:
-        torch.Tensor: spatial gradients
-
-    Shape:
-        - Input: :math:`(B, C, H, W)`
-        - Output: :math:`(B, C, 3, H, W)`
-
-    """
-    b, c, h, w = x.shape
-    ksize = get_gausskernel_size(sigma)
-    out =  torch.zeros(b,c,3,h,w)
-    return out
-
 def affine(center: torch.Tensor, unitx: torch.Tensor, unity: torch.Tensor) -> torch.Tensor:
     r"""Computes transformation matrix A which transforms point in homogeneous coordinates from canonical coordinate system into image
 
@@ -139,3 +125,47 @@ def extract_affine_patches(input: torch.Tensor,
     out =  torch.zeros(num_patches, ch, PS, PS)
     return out
 
+
+def extract_antializased_affine_patches(input: torch.Tensor,
+                           A: torch.Tensor,
+                           img_idxs: torch.Tensor,
+                           PS: int = 32,
+                           ext: float = 6.0):
+    """Extract patches defined by affine transformations A from scale pyramid created image tensor X.
+    It runs your implementation of the `extract_affine_patches` function, so it would not work w/o it.
+    You do not need to ever modify this finction, implement `extract_affine_patches` instead.
+    
+    Args:
+        input: (torch.Tensor) images, :math:`(B, CH, H, W)`
+        A: (torch.Tensor). :math:`(N, 3, 3)`
+        img_idxs: (torch.Tensor). :math:`(N, 1)` indexes of image in batch, where patch belongs to
+        PS: (int) output patch size in pixels, default = 32
+        ext (float): output patch size in unit vectors. 
+
+    Returns:
+        patches: (torch.Tensor) :math:`(N, CH, PS,PS)`
+    """
+    import kornia
+    b,ch,h,w = input.size()
+    num_patches = A.size(0)
+    scale = (kornia.feature.get_laf_scale(ext * A.unsqueeze(0)[:,:,:2,:]) / float(PS))[0]
+    half: float = 0.5
+    pyr_idx = (scale.log2()).relu().long()
+    cur_img = input
+    cur_pyr_level = 0
+    out = torch.zeros(num_patches, ch, PS, PS).to(device=A.device, dtype=A.dtype)
+    while min(cur_img.size(2), cur_img.size(3)) >= PS:
+        _, ch_cur, h_cur, w_cur = cur_img.size()
+        scale_mask = (pyr_idx == cur_pyr_level).squeeze()
+        if (scale_mask.float().sum()) >= 0:
+            scale_mask = (scale_mask > 0).view(-1)
+            current_A = A[scale_mask]
+            current_A[:, :2, :3] *= (float(h_cur)/float(h))
+            patches = extract_affine_patches(cur_img,
+                                 current_A, 
+                                 img_idxs[scale_mask],
+                                 PS, ext)
+            out.masked_scatter_(scale_mask.view(-1, 1, 1, 1), patches)
+        cur_img = kornia.geometry.pyrdown(cur_img)
+        cur_pyr_level += 1
+    return out

assignment_0_3_correspondences_template/local_descriptor.py

@@ -4,12 +4,12 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import typing
-
+from typing import Tuple
 from imagefiltering import * 
 from local_detector import *
 
 
-def affine_from_location(b_ch_d_y_x: torch.Tensor)-> torch.Tensor:
+def affine_from_location(b_ch_d_y_x: torch.Tensor)-> Tuple[torch.Tensor, torch.Tensor]:
     r"""Computes transformation matrix A which transforms point in homogeneous coordinates from canonical coordinate system into image
     from keypoint location (output of scalespace_harris or scalespace_hessian)
     Return:
@@ -26,7 +26,7 @@ def affine_from_location(b_ch_d_y_x: torch.Tensor)-> torch.Tensor:
 
 
 def affine_from_location_and_orientation(b_ch_d_y_x: torch.Tensor,
-                                         ori: torch.Tensor)-> torch.Tensor:
+                                         ori: torch.Tensor)-> Tuple[torch.Tensor, torch.Tensor]:
     r"""Computes transformation matrix A which transforms point in homogeneous coordinates from canonical coordinate system into image
     from keypoint location (output of scalespace_harris or scalespace_hessian). Ori - orientation angle in radians
     Return:
@@ -44,7 +44,7 @@ def affine_from_location_and_orientation(b_ch_d_y_x: torch.Tensor,
 
 def affine_from_location_and_orientation_and_affshape(b_ch_d_y_x: torch.Tensor,
                                                       ori: torch.Tensor,
-                                                      aff_shape: torch.Tensor)-> torch.Tensor:
+                                                      aff_shape: torch.Tensor)-> Tuple[torch.Tensor, torch.Tensor]:
     r"""Computes transformation matrix A which transforms point in homogeneous coordinates from canonical coordinate system into image
     from keypoint location (output of scalespace_harris or scalespace_hessian)
     Return:

assignment_0_3_correspondences_template/local_detector.py

@@ -5,37 +5,6 @@ import torch.nn.functional as F
 import typing
 from imagefiltering import * 
 
-
-def hessian_response(x: torch.Tensor, sigma: float)-> torch.Tensor:
-    """Computes the determinant of the Hessian matrix.
-    The response map is computed according the following formulation:
-    .. math::
-        R = det(H)
-    where:
-
-    .. math::
-        M = \sum_{(x,y) \in W}
-        \begin{bmatrix}
-            I_{xx} & I_{xy} \\
-            I_{xy} & I_{yy} \\
-        \end{bmatrix}
-
-    Args:
-        x: torch.Tensor: 4d tensor
-        sigma (float): sigma of Gaussian derivative
-
-    Return:
-        torch.Tensor: Hessian response
-
-    Shape:
-       - Input: :math:`(B, C, H, W)`
-       - Output: :math:`(B, C, H, W)`
-    """
-    out = torch.zeros_like(x)
-    return out
-
-
-
 def harris_response(x: torch.Tensor,
                      sigma_d: float,
                      sigma_i: float,
@@ -89,24 +58,6 @@ def nms2d(x: torch.Tensor, th: float = 0):
     out = torch.zeros_like(x)
     return out
 
-def hessian(x: torch.Tensor, sigma: float, th: float = 0):
-    r"""Returns the coordinates of maximum of the Hessian function.
-    Args:
-        x: torch.Tensor: 4d tensor
-        sigma (float): scale
-        th (float): threshold
-
-    Return:
-        torch.Tensor: coordinates of local maxima in format (b,c,h,w)
-
-    Shape:
-      - Input: :math:`(B, C, H, W)`
-      - Output: :math:`(N, 4)`, where N - total number of maxima and 4 is (b,c,h,w) coordinates
-    """
-    # To get coordinates of the responces, you can use torch.nonzero function
-    out = torch.zeros(0,2)
-    return out
-
 
 def harris(x: torch.Tensor, sigma_d: float, sigma_i: float, th: float = 0):
     r"""Returns the coordinates of maximum of the Harris function.
@@ -160,22 +111,6 @@ def nms3d(x: torch.Tensor, th: float = 0):
     return out
 
 
-def scalespace_hessian_response(x: torch.Tensor,
-                                n_levels: int = 40,
-                                sigma_step: float = 1.1):
-    r"""First computes scale space and then computes the determinant of Hessian matrix on 
-    Args:
-        x: torch.Tensor: 4d tensor
-        n_levels (int): number of the levels, (default 40)
-        sigma_step (float): blur step, (default 1.1)
-
-    Shape:
-      - Input: :math:`(B, C, H, W)`
-      - Output: :math:`(B, C, N_LEVELS, H, W)`, List(floats)
-    """
-    out = torch.zeros(b, ch, n_levels, h, w), [1.0 for x in range(n_levels)]
-    return out
-
 
 def scalespace_harris_response(x: torch.Tensor,
                                 n_levels: int = 40,
@@ -194,26 +129,6 @@ def scalespace_harris_response(x: torch.Tensor,
     return out
 
 
-def scalespace_hessian(x: torch.Tensor,
-                       th: float = 0,
-                       n_levels: int = 40,
-                       sigma_step: float = 1.1):
-    r"""Returns the coordinates of maximum of the Hessian function.
-    Args:
-        x: torch.Tensor: 4d tensor
-        th (float): threshold
-        n_levels (int): number of scale space levels (default 40)
-        sigma_step (float): blur step, (default 1.1)
-        
-    Shape:
-      - Input: :math:`(B, C, H, W)`
-      - Output: :math:`(N, 5)`, where N - total number of maxima and 5 is (b,c,d,h,w) coordinates
-    """
-    # To get coordinates of the responces, you can use torch.nonzero function
-    # Don't forget to convert scale index to scale value with use of sigma
-    out = torch.zeros(0,3)
-    return out
-
 
 def scalespace_harris(x: torch.Tensor,
                        th: float = 0,

assignment_0_3_correspondences_template/matching.py

@@ -5,21 +5,6 @@ import torch.nn.functional as F
 import typing
 
 
-def match_nn(desc1: torch.Tensor, desc2: torch.Tensor):
-    '''Function, which finds nearest neightbors for each vector in desc1.
-    
-    Return:
-        torch.Tensor: indexes of matching descriptors in desc1 and desc2
-        torch.Tensor: L2 desriptor distance
-
-
-    Shape:
-        - Input :math:`(B1, D)`, :math:`(B2, D)`
-        - Output: :math:`(B1, 2)`, :math:`(B1, 1)`
-    '''
-    matches_idxs = torch.arange(0, desc2.size(0)).view(-1, 1).repeat(1, 2)
-    match_dists = torch.zeros(desc2.size(0),1)
-    return matches_idxs, match_dists
 
 
 def match_snn(desc1: torch.Tensor, desc2: torch.Tensor, th: float = 0.8):
@@ -38,40 +23,3 @@ def match_snn(desc1: torch.Tensor, desc2: torch.Tensor, th: float = 0.8):
     matches_idxs = torch.arange(0, desc2.size(0)).view(-1, 1).repeat(1, 2)
     match_dists = torch.zeros(desc2.size(0),1)
     return matches_idxs, match_dists
-
-def match_mnn(desc1: torch.Tensor, desc2: torch.Tensor):
-    '''Function, which finds mutual nearest neightbors for each vector in desc1 and desc2,
-    which satisfies first to second nearest neighbor distance <= th check
-    
-    Return:
-        torch.Tensor: indexes of matching descriptors in desc1 and desc2
-        torch.Tensor: L2 desriptor distance
-
-
-    Shape:
-        - Input :math:`(B1, D)`, :math:`(B2, D)`
-        - Output: :math:`(B3, 2)`, :math:`(B3, 1)` where 0 <= B3 <= min(B1,B2)
-    '''
-    matches_idxs = torch.arange(0, desc2.size(0)).view(-1, 1).repeat(1, 2)
-    match_dists = torch.zeros(desc2.size(0),1)
-    return matches_idxs, match_dists
-
-def match_smnn(desc1: torch.Tensor, desc2: torch.Tensor, th: float = 0.8):
-    '''Function, which finds mutual nearest neightbors for each vector in desc1 and desc2,
-    which satisfy first to second nearest neighbor distance <= th check in both directions. 
-    So, it is intersection of match_mnn(d1,d2), match_snn(d1,d2), match_snn(d2,d1)
-    Resulting distance ratio should be maximum over over distance ratio in both directions
-    
-    Return:
-        torch.Tensor: indexes of matching descriptors in desc1 and desc2
-        torch.Tensor: L2 desriptor distance ratio 1st to 2nd nearest neighbor
-
-
-    Shape:
-        - Input :math:`(B1, D)`, :math:`(B2, D)`
-        - Output: :math:`(B3, 2)`, :math:`(B3, 1)`, where 0 <= B3 <= min(B1, B2)
-    '''
-    matches_idxs = torch.arange(0, desc2.size(0)).view(-1, 1).repeat(1, 2)
-    match_dists = torch.zeros(desc2.size(0),1)
-    return matches_idxs, match_dists
-

assignment_0_3_correspondences_template/oriviz.py

+import pdb, torch
+import time
+import os
+import math
+from torchvision import transforms
+import cv2
+import kornia
+import numpy as np
+from IPython.display import HTML, clear_output
+import IPython.display
+import matplotlib.pyplot as plt
+import matplotlib as mplt
+import matplotlib.image as mpimg
+from celluloid import Camera 
+import numpy
+import ipywidgets as widgets
+import tkinter as tk
+from tkinter import ttk
+from ipywidgets import interact, interactive, fixed, interact_manual
+import mpl_interactions.ipyplot as iplt
+import time
+import matplotlib.image as mpimg
+import matplotlib.patches as mpatches
+from kornia.feature import get_laf_orientation, set_laf_orientation, extract_patches_from_pyramid
+
+FULL_ROTATION = 360
+SIZE_IMG = 150
+
+def play_with_angle(img, A, orientation_estimation):
+  """
+  Interactive visualization(with the slider) of working of your orientation_estimation function.
+
+  Args:
+    patch: (torch.Tensor) 
+    orientation_estimation: estimator function
+
+  Returns:
+    nothing, but as side affect patches are shown  
+  """
+  laf = A[:,:2,:].reshape(1,1,2,3)
+  orig_angle = get_laf_orientation(laf)
+  patch = extract_patches_from_pyramid(img, laf)[0]
+  
+  patch = kornia.tensor_to_image(patch)
+  fig, ax = plt.subplots(1, 3, figsize=(12, 4))
+  ax1 = ax[0].imshow(patch, cmap='gray')
+  ax2 = ax[1].imshow(patch, cmap='gray')
+  ax3 = ax[2].imshow(patch, cmap='gray')
+
+  ax[0].set_title("Rotated patch")
+  ax[1].set_title("user normalized patch")
+  ax[2].set_title("kornia normalized patch")
+   
+    
+  plt.close()
+
+  slider = widgets.FloatSlider(value=0, min=0, max=360, step=1, description="Angle:")
+  widgets.interact(img_viz, img=fixed(img), A=fixed(A), orientation_estimation=fixed(orientation_estimation), fig=fixed(fig), ax1=fixed(ax1), ax2=fixed(ax2), ax3=fixed(ax3), alfa=slider)
+
+
+# helper function. It is called as a parametr of widgets.interact()
+def img_viz(img: torch.tensor, A: torch.tensor, orientation_estimation, fig, ax1, ax2, ax3, alfa=0):
+  alfa = alfa
+  laf = A[:,:2,:].reshape(1,1,2,3)
+  orig_angle = get_laf_orientation(laf)
+  patch = extract_patches_from_pyramid(img, laf)
+  angle = torch.tensor([np.float32(alfa)])
+  laf_current = set_laf_orientation(laf, alfa + orig_angle)
+    
+  patch_rotated = extract_patches_from_pyramid(img, laf_current)[0]
+
+  grad_ori = kornia.feature.orientation.PatchDominantGradientOrientation(32)
+  estimated_angle = -orientation_estimation(patch_rotated).reshape(-1)
+  estimated_angle_kornia = grad_ori(patch_rotated).reshape(-1)
+
+  prev_angle = get_laf_orientation(laf_current)
+  laf_out_user = set_laf_orientation(laf_current, torch.rad2deg(estimated_angle) + prev_angle)
+  laf_out_kornia = set_laf_orientation(laf_current, torch.rad2deg(estimated_angle_kornia) + prev_angle)
+
+  patch_out = extract_patches_from_pyramid(img, laf_out_user).reshape(1,1,32,32)
+  patch_out_kornia = extract_patches_from_pyramid(img, laf_out_kornia).reshape(1,1,32,32)
+
+  img1 = kornia.tensor_to_image(patch_rotated)
+  img2 = kornia.tensor_to_image(patch_out)
+  img3 = kornia.tensor_to_image(patch_out_kornia)
+    
+
+  ax1.set_data(img1)
+  ax2.set_data(img2)
+  ax3.set_data(img3)
+  display(fig)
+  plt.close()
+
+
+""" The SIFT visualization """
+
+
+
+# Visualization 
+
+def reshape_to_bins(siftdesc, spatial_dim, orient_dim):
+  b, dim = siftdesc.shape
+  return siftdesc.reshape(b, orient_dim, -1).reshape(b, orient_dim, spatial_dim, spatial_dim)
+
+def visualize_sift(desc,  orient_dim=8, spatial_dim=4, title = '', is_absolute_mode =1):
+  res = 5*reshape_to_bins(desc, spatial_dim, orient_dim)
+  max = -1.0
+  angle = 360.0 / orient_dim
+  alfa = 0
+
+  # finding the maximum number in the res tensor
+  for i in range(0, spatial_dim):
+    for j in range(0, spatial_dim):
+      for k in range(0, orient_dim):
+        if res[:, :, i, j][0][k] < 0:
+          sys.exit("Provided tensor cannot contain a negative number!")
+        max = res[:, :, i, j][0][k] if (res[:, :, i, j][0][k] > max) else max
+
+  max = float(max)
+
+  # creating the table
+  f, ax = plt.subplots(spatial_dim, spatial_dim, figsize=(10, 10), gridspec_kw = {'wspace':0, 'hspace':0})
+
+  if is_absolute_mode == 1:
+    f.suptitle(f"{title} bins abs histogram", fontsize=16)
+  else:
+    f.suptitle(f"{title} bins rel values", fontsize=16)
+
+  plt.subplots_adjust(wspace=0, hspace=0)
+  plt.rcParams['axes.linewidth'] = 5
+
+  # colorizing the table 
+  for i in range(spatial_dim):
+    for j in range(spatial_dim):
+      axx = ax
+      if spatial_dim != 1:
+        axx = ax[i][j]
+      axx.spines['bottom'].set_color('green')
+      axx.spines['top'].set_color('green')
+      axx.spines['right'].set_color('green')
+      axx.spines['left'].set_color('green')
+      axx.set_xticklabels([])
+      axx.set_yticklabels([])
+
+
+  # drawing the arrows corresponding the elements of the res tensor
+  for i in range(0, spatial_dim):
+    for j in range(0, spatial_dim):
+      for k in range(0, orient_dim):
+        if float(res[:, :, i, j][0][k]) == 0:
+          alfa += angle
+          continue
+        ratio = max / float(res[:, :, i, j][0][k])
+        if ratio <= 10 or is_absolute_mode == 1:
+          ratio = float(res[:, :, i, j][0][k]) if is_absolute_mode == 1 else float(res[:, :, i, j][0][k]) / max
+          axx = ax
+          if spatial_dim != 1:
+            axx = ax[i][j]
+          if alfa >= 0 and alfa <= 90:
+            axx.add_patch(mpatches.FancyArrowPatch((1, 1), (1 + ratio * math.cos(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item()), 1 + ratio * math.sin(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item())), mutation_scale=5))
+          elif alfa > 90 and alfa <= 180:
+            tmp = alfa
+            alfa = 180 - alfa
+            axx.add_patch(mpatches.FancyArrowPatch((1, 1), (1 - ratio * math.cos(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item()), 1 + ratio * math.sin(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item())), mutation_scale=5))
+            alfa = tmp         
+          elif alfa > 180 and alfa <= 270:
+            tmp = alfa
+            alfa -= 180
+            axx.add_patch(mpatches.FancyArrowPatch((1, 1), (1 - ratio * math.cos(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item()), 1 - ratio * math.sin(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item())), mutation_scale=5))
+            alfa = tmp          
+          else: 
+            tmp = alfa
+            alfa = 360 - alfa
+            axx.add_patch(mpatches.FancyArrowPatch((1, 1), (1 + ratio * math.cos(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item()),  1 - ratio * math.sin(kornia.geometry.conversions.deg2rad(torch.tensor(alfa)).item())), mutation_scale=5))
+            alfa = tmp
+        alfa += angle     
+      axx.set(xlim=(0, 2), ylim=(0, 2))
+      alfa = 0
+  return f 

assignment_0_3_correspondences_template/wbs.py

-import numpy as np
-import math
-import torch
-import torch.nn.functional as F
-import typing
-
-from imagefiltering import * 
-from local_detector import *
-from local_descriptor import *
-from ransac import *
-from matching import *
-
-
-
-def detect_and_describe(img, *kwargs):
-    return
-
-def match_and_filter(descs1, descs2):
-    return
-
-