no opencv version check

e233e616 · Dmytro Mishkin · 117bdb6a · e233e616
Commit e233e616 authored 3 years ago by Dmytro Mishkin
--- a/assignment_0_3_correspondences_template/image-matching-interactive.ipynb
+++ b/assignment_0_3_correspondences_template/image-matching-interactive.ipynb
@@ -69,8 +69,8 @@
    "import os\n",
    "DBL_EPS = np.finfo(float).eps\n",
    "\n",
-    "major, minor, *_ = [int(v) for v in cv2.__version__.split('.')]\n",
-    "assert major >= 4 and minor >= 5, f\"Please update your cv2. Required >= 4.5.0, yours: {cv2.__version__}\""
+    "#major, minor, *_ = [int(v) for v in cv2.__version__.split('.')]\n",
+    "#assert major >= 4 and minor >= 5, f\"Please update your cv2. Required >= 4.5.0, yours: {cv2.__version__}\""
   ]
  },
  {
@@ -723,7 +723,7 @@
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "16b425b9f5a542a097fb32df7490fd0a",
+       "model_id": "5f36938936e44db59cf6827d573b8daf",
       "version_major": 2,
       "version_minor": 0
      },

 %% Cell type:markdown id: tags:

 ![image.png](wide-baseline-stereo-demo_files/att_00000.png)

 %% Cell type:markdown id: tags:

 ## Interactive image matching pipeline

 This is a helper jupyter notebook, which contains [interactive widgets](https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20Basics.html) for qualitative evaluation of the image matching pipeline and better understanding of the robust model fitting with RANSAC. You can plug-in the functions, which you have implemented, or use the pre-defined wrappers around OpenCV or kornia functions.

 It also can be useful as a template for writing your own visualizations or interactive apps.

 %% Cell type:markdown id: tags:

 Let's first do in in OpenCV to get the high-level understanding of the steps. We have two image as an input and would like to get corresponces and perspective transform between them.

 %% Cell type:markdown id: tags:

 ## Recommended way of using this notebook

 This notebook is a helper tool for you to check if the modules you have developed are working in practice. For doing this, you select the appropriate function in the interactive section below. It would not be shown, until you run all the cells, as the interactive widgets are not persistent.

 You can also find and modify if you want, the commented implementation of the interactive part.

 %% Cell type:code id: tags:

 ``` python
 %matplotlib inline
 import matplotlib.pyplot as plt
 import numpy as np
 import seaborn as sns
 import numpy as np
 import cv2
 import math
 import random
 import torch
 import torch.nn as nn
 from scipy.stats import norm, uniform
 import matplotlib.colors as mcolors
 import matplotlib.pyplot as plt
 from ipywidgets import Button, HBox, VBox, Layout
 import ipywidgets as wd
 from IPython.display import YouTubeVideo
 from typing import List, Dict, Tuple, Callable
 from pathlib import Path
 import sys
 from collections import namedtuple
 from copy import deepcopy
 import kornia as K
 import kornia.feature as KF
 from kornia_moons.feature import *
 import os
 DBL_EPS = np.finfo(float).eps

-major, minor, *_ = [int(v) for v in cv2.__version__.split('.')]
-assert major >= 4 and minor >= 5, f"Please update your cv2. Required >= 4.5.0, yours: {cv2.__version__}"
+#major, minor, *_ = [int(v) for v in cv2.__version__.split('.')]
+#assert major >= 4 and minor >= 5, f"Please update your cv2. Required >= 4.5.0, yours: {cv2.__version__}"
 ```

 %% Cell type:code id: tags:

 ``` python
 # --------------------- jupyter notebook only START ---------------------
 if Path().absolute().name == "RANSAC":
    os.chdir(str(Path().absolute().parent))

 from RANSAC.plot_planar import *
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python

 def timg_load(filename):
    """ load an image, return a tensor image. """
    img = cv2.imread(filename)
    with torch.no_grad():
        timg = K.image_to_tensor(img, False).float() / 255
        timg = K.color.bgr_to_grayscale(timg)
    return timg

 def img_to_timg(timg):
    timg = K.image_to_tensor(img1, False).float()/255.
    if timg.shape[-3] == 3:
        timg = K.color.rgb_to_grayscale(timg)
    return timg
 ```

 %% Cell type:code id: tags:

 ``` python
 rr = {}
 rr['name'] = "sdsd"
 ```

 %% Cell type:code id: tags:

 ``` python
 rr
 ```

 %% Output

    {'name': 'sdsd'}

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 Detections = namedtuple('Detections', ['kps1',
                                       'kps2',
                                       'tentative_matches',
                                       'pts_matches',
                                       'H',
                                       'inlier_mask',
                                       'lafs1',
                                       'lafs2'])
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:markdown id: tags:

 ## DETECT, ORIENT, AFFINE, DESCRIBE, MATCH, RANSAC

 ### OUR PIPELINE

 %% Cell type:code id: tags:

 ``` python
 from typing import Dict
 class CustomLocalFeatureMatcher():
    def __init__(self):
        return
    def detectAndMatch(self,
                       timg1: torch.Tensor,
                       timg2: torch.Tensor,
                       detector: str = 'kornia Harris',
                       orientation: str = 'kornia orientation',
                       affine: str = 'none',
                       descriptor: str = 'kornia SIFT',
                       match: str = 'kornia snn',
                       ransac: str = 'kornia RANSAC') -> Detections:
        device = torch.device('cpu')

        ### Detector
        if detector == 'kornia Harris':
            detector = KF.ScaleSpaceDetector(2000,
                              resp_module=KF.CornerGFTT(),
                              nms_module=K.geometry.subpix.ConvQuadInterp3d(10, 1e-5),
                              scale_pyr_module=K.geometry.ScalePyramid(3, 1.6, 32, double_image=False),
                              ori_module=KF.PassLAF(),
                              aff_module=KF.PassLAF(),
                              mr_size=6.0).to(device)
            lafs1, resps1 = detector(timg1, None)
            lafs2, resps2 = detector(timg2, None)
        elif detector == 'kornia Hessian':
            detector = KF.ScaleSpaceDetector(2000,
                              resp_module=KF.BlobHessian(),
                              nms_module=K.geometry.subpix.ConvQuadInterp3d(10, 1e-5),
                              scale_pyr_module=K.geometry.ScalePyramid(3, 1.6, 32, double_image=False),
                              ori_module=KF.PassLAF(),
                              aff_module=KF.PassLAF(),
                              mr_size=6.0).to(device)
            lafs1, resps1 = detector(timg1, None)
            lafs2, resps2 = detector(timg2, None)
        elif detector == 'assignment Hessian':
            from local_detector import scalespace_hessian
            # The threshold used here are for the reference implementation.
            # Please, adjust them for your implementation
            out1 = scalespace_hessian(timg1, 0.0001, 10, 1.3)
            mrSize = 5.0
            num_kpts = len(out1)
            lafs1 = KF.laf_from_center_scale_ori(out1[:, 3:].reshape(1, -1, 2).flip(2),
                                                 mrSize * out1[:, 2:3].reshape(1, -1, 1, 1),
                                                 torch.zeros(1, num_kpts, 1))

            # The threshold used here are for the reference implementation.
            # Please, adjust them for your implementation
            out2 = scalespace_hessian(timg2, 0.0001, 10, 1.3)
            num_kpts2 = len(out2)
            lafs2 = KF.laf_from_center_scale_ori(out2[:, 3:].reshape(1, -1, 2).flip(2),
                                                 mrSize * out2[:, 2:3].reshape(1, -1, 1, 1),
                                                 torch.zeros(1, num_kpts2, 1))
        elif detector == 'assignment Harris':
            from local_detector import scalespace_harris
            # The threshold used here are for the reference implementation.
            # Please, adjust them for your implementation
            mrSize = 5.0

            out1 = scalespace_harris(timg1, 0.000001, 10, 1.3)
            num_kpts = len(out1)
            lafs1 = KF.laf_from_center_scale_ori(out1[:, 3:].reshape(1, -1, 2).flip(2),
                                                 mrSize *out1[:, 2:3].reshape(1, -1, 1, 1),
                                                 torch.zeros(1, num_kpts, 1))

            # The threshold used here are for the reference implementation.
            # Please, adjust them for your implementation
            out2 = scalespace_harris(timg2, 0.000001, 10, 1.3)
            num_kpts2 = len(out2)
            lafs2 = KF.laf_from_center_scale_ori(out2[:, 3:].reshape(1, -1, 2).flip(2),
                                                 mrSize * out2[:, 2:3].reshape(1, -1, 1, 1),
                                                 torch.zeros(1, num_kpts2, 1))
        else:
            raise ValueError ("Unknown detector")

        ### Optional affine shape estimation

        if affine == 'kornia affine':
            aff = KF.LAFAffineShapeEstimator(19)
            lafs1 = aff(lafs1, timg1)
            lafs2 = aff(lafs2, timg2)
        elif affine == 'assignment affine':
            from local_descriptor import estimate_patch_affine_shape
            def custom_aff(x):
                return estimate_patch_affine_shape(x).unsqueeze(1)
            aff = KF.LAFAffineShapeEstimator(19, affine_shape_detector=custom_aff)
            lafs1 = aff(lafs1, timg1)
            lafs2 = aff(lafs2, timg2)
        elif affine == 'none':
            pass
        else:
            raise ValueError ("Unknown orientation")

        ### Orientation estimation

        if orientation == 'kornia orientation':
            ori = KF.LAFOrienter(19, 36)
            lafs1 = ori(lafs1, timg1)
            lafs2 = ori(lafs2, timg2)
        elif orientation == 'assignment orientation':
            from local_descriptor import estimate_patch_dominant_orientation
            ori = KF.LAFOrienter(19, angle_detector=estimate_patch_dominant_orientation)
            lafs1 = ori(lafs1, timg1)
            lafs2 = ori(lafs2, timg2)
        elif orientation == 'none':
            pass
        else:
            raise ValueError ("Unknown orientation")

        ### Descriptor
        if descriptor == 'kornia SIFT':
            desc_module = KF.LAFDescriptor(KF.SIFTDescriptor(32, rootsift=True), 32)

            desc1 = desc_module(timg1, lafs1).reshape(lafs1.shape[1], -1)
            desc2 = desc_module(timg2, lafs2).reshape(lafs2.shape[1], -1)
        elif descriptor == 'assignment SIFT':
            from local_descriptor import calc_sift_descriptor
            class TempDesc(nn.Module):
                def forward(self, x):
                    return calc_sift_descriptor(x)
            desc_module = KF.LAFDescriptor(TempDesc(), 32)
            desc1 = desc_module(timg1, lafs1).reshape(lafs1.shape[1], -1)
            desc2 = desc_module(timg2, lafs2).reshape(lafs2.shape[1], -1)
        else:
            raise ValueError("Unknown descriptor")

        ### Matching
        if match == 'kornia snn':
            dists, idxs = KF.match_snn(desc1, desc2, 0.9)
        elif match == 'assignment snn':
            from matching import match_snn
            dists, idxs = match_snn(desc1, desc2, 0.9)
        else:
            raise ValueError("Unknown matching")

        pts_matches1 = KF.get_laf_center(lafs1[:, idxs[:, 0]]).view(-1, 2)
        pts_matches2 = KF.get_laf_center(lafs2[:, idxs[:, 1]]).view(-1, 2)

        if ransac == 'kornia RANSAC':
            RANSAC =  K.geometry.RANSAC('homography', 5.0, 1024, 5)
            H, inliers_mask = RANSAC(pts_matches1, pts_matches2)
        elif ransac == 'assignment RANSAC':
            from ransac import ransac_h
            H, inliers_mask = ransac_h(torch.cat([pts_matches1, pts_matches2], dim=1))
        else:
            raise ValueError("Unknown RANSAC")

        kps1 = KF.get_laf_center(lafs1).view(-1, 2)
        kps2 = KF.get_laf_center(lafs2).view(-1, 2)

        result = Detections(kps1,
                            kps2,
                            idxs,
                            torch.cat([pts_matches1, pts_matches2],dim=1),
                            H,
                            inliers_mask,
                            lafs1,
                            lafs2)


        return result
 ```

 %% Cell type:markdown id: tags:

 Here we will load a couple of images to test

 %% Cell type:code id: tags:

 ``` python
 timg1 = timg_load('v_woman1.ppm')
 timg2 = timg_load('v_woman6.ppm')

 img1 = K.tensor_to_image(255*timg1).astype(np.uint8)
 img2 = K.tensor_to_image(255*timg2).astype(np.uint8)
 ```

 %% Cell type:markdown id: tags:

 Let's create the choices for the drop-down list. They should match whatever we have inside `CustomLocalFeatureMatcher` above.

 %% Cell type:code id: tags:

 ``` python
 possible_detectors = ["kornia Harris",
                      "kornia Hessian",
                      "assignment Harris",
                      "assignment Hessian"]
 possible_ori = ["none",
                "kornia orientation",
                "assignment orientation"]

 possible_affine = ["none",
                   "kornia affine",
                   "assignment affine"]

 possible_descriptor = ["kornia SIFT",
                       "assignment SIFT"]

 possible_matching = ["kornia snn",
                     "assignment snn"]

 possible_ransac = ["kornia RANSAC",
                   "assignment RANSAC"]

 ```

 %% Cell type:markdown id: tags:

 ## PARAMETRIZATION

 Throughout the whole notebook, we use data classes to parametrize algorithms. They are easy to use. They are extensible and clean. We encourage you to add new parameters and adjust our code to your needs. You can change the detection, description, and matching pipeline completely or use our and tweak the parameters.

 ## Dataclasses

 We defined multiple data classes RansacPlanarFunctions, RansacPlanarParams, and PlotParams. We use them for storing values and parametrization. Their advantage over simple dictionaries is that they have defined values, are comparable, and [many others](https://docs.python.org/3/library/dataclasses.html). We also use them to store RANSAC output.

 Below, you can see how do we define them within the hidden codebase. You can change all of the values below in the initialization section. (or during runtime)

 %% Cell type:code id: tags:

 ``` python
 # Here are default parameters for out app.
 @dataclass(eq=False)
 class WxBSParams:
    detector: str = 'kornia Harris'
    orientation: str = 'kornia orientation'
    affine: str = 'none'
    descriptor: str = 'kornia SIFT'
    match: str = 'kornia snn'
    ransac: str = 'kornia RANSAC'
 ```

 %% Cell type:code id: tags:

 ``` python
 matching_params = WxBSParams()
 plt_params = PlotParams()
 ```

 %% Cell type:markdown id: tags:

 ## Global variables and support plotting functions

 %% Cell type:code id: tags:

 ``` python
 # Modes in which inliers/outliers are displayed.
 # Modes works as follows: there are three modes: 0, 1, 2.
 #                         Mode 0: do not plot anything
 #                         Mode 1: show only keypoints (in both images)
 #                         Mode 2: show regions (in both images)
 #                         Mode 3: show keypoints and connect them with lines
 #                         Mode 4: show regions and connect them with lines

 visualization_options = {
    "inliers_display_mode": 4,
    "outliers_display_mode": 1,
 }


 # buttons that control the run of the algorithm
 alg_buttons = [
    wd.Button(description="detect"),
    wd.Button(description="match"),

 ]

 det_selector = wd.Dropdown(description='Detector',
                           options=possible_detectors,
                           value=possible_detectors[0],
                           layout=Layout(width='80%'))

 ori_selector = wd.Dropdown(description='Orientation',
                           options=possible_ori,
                           value=possible_ori[0],
                           layout=Layout(width='80%'))

 affine_selector = wd.Dropdown(description='Affine',
                           options=possible_affine,
                           value=possible_affine[0],
                           layout=Layout(width='80%'))


 descriptor_selector = wd.Dropdown(description='Descriptor',
                           options=possible_descriptor,
                           value=possible_descriptor[0],
                           layout=Layout(width='80%'))
 matching_selector = wd.Dropdown(description='Matching',
                           options=possible_matching,
                           value=possible_matching[0],
                           layout=Layout(width='80%'))

 ransac_selector = wd.Dropdown(description='RANSAC',
                           options=possible_ransac,
                           value=possible_ransac[0],
                           layout=Layout(width='80%'))

 # buttons that control the visualizations
 visu_buttons = [
    wd.Button(description=f"inl display mode: {visualization_options['inliers_display_mode']}"),
    wd.Button(description=f"out display mode: {visualization_options['outliers_display_mode']}"),
 ]

 dropdowns = [
    det_selector,
    ori_selector,
    affine_selector,
    descriptor_selector,
    matching_selector,
    ransac_selector]

 def disable_buttons(func: Callable):
    """ Decorator that disables all buttons during the run of the function func """
    def disable_enable(*args, **kwargs):
        global visu_buttons, alg_buttons
        buttons = visu_buttons + alg_buttons
        for btn in buttons:
            btn.disabled = True
        result = func(*args, **kwargs)
        for btn in buttons:
            btn.disabled = False
        return result
    return disable_enable


 def output_img_update(img):
    """ Update the output_img. Display the image img."""
    with output_img:
        output_img.clear_output(True)
        plt.figure(figsize=(20, 10))
        plt.axis('off')
        plt.imshow(img)
        plt.show()


 def show_keypoints():
    """ Show only keypoints."""
    global img1, img2, detections, plt_params
    img = draw_keypoints(img1, img2, detections.lafs1, detections.lafs2, plt_params)
    output_img_update(img)


 @disable_buttons
 def reset_H_keypoints_show(_=None):
    """
    Reset both current and the best RANSAC run.
    Forget all homographies, inliers, outliers, etc., and show only keypoints.
    """
    global detections, img1, img2, plt_params, timg1, timg2, params, functions, log, best_log
    log = RansacPlanarLog()
    best_log = RansacPlanarLog()
    show_keypoints()
    with output_legend:
        output_legend.clear_output(True)
        output_best_legend.clear_output(True)
        plot_legend_init(detections.kps1, detections.kps2, plt_params)


 @disable_buttons
 def run_detector_show(_=None):
    """ Reset all RANSAC runs and run the detect/describe/match pipeline. """
    global detections, img1, img2, plt_params, timg1, timg2, params, functions, matching_params
    cfm = CustomLocalFeatureMatcher()
    with torch.no_grad():
        detections = cfm.detectAndMatch(timg1, timg2,
                                 det_selector.value,
                                 ori_selector.value,
                                 affine_selector.value,
                                 descriptor_selector.value,
                                 matching_selector.value,
                                 ransac_selector.value)
    reset_H_keypoints_show()


 def output_legend_update(_=None):
    """ Update both current output legend and the best output legend """
    global log, best_log, detections, plt_params, output_legend, output_best_legend
    with output_legend:
        output_legend.clear_output(True)
        plot_current_legend(log, detections, plt_params)


 def show_matches():
    """ Show current inliers, outliers, homography, update legends etc. """
    global img1, img2, detections, log, plt_params, best_log
    img = draw_matches(img1, img2, detections, plt_params, visualization_options)
    output_img_update(img)
    output_legend_update()


 def inliers_display_mode(_=None):
    """ Cycle the inliers display mode """
    global visualization_options, log, best_log, visu_buttons
    visualization_options["inliers_display_mode"] = (visualization_options["inliers_display_mode"]+1)%4
    visu_buttons[0].description = f"inl display mode: {visualization_options['inliers_display_mode']}"
    if detections.H is None:
        show_keypoints()
    else:
        show_matches()


 def outliers_display_mode(_=None):
    """ Cycle the outliers display mode """
    global visualization_options, log, best_log, visu_buttons
    visualization_options["outliers_display_mode"] = (visualization_options["outliers_display_mode"]+1)%4
    visu_buttons[1].description =f"out display mode: {visualization_options['outliers_display_mode']}"
    if detections.H is None:
        show_keypoints()
    else:
        show_matches()



 @disable_buttons
 def ransac_fit_show(_=None):
    """ Run the complete RANSAC algorithm, and show the result. """
    global detections, plt_params, params, functions, log, img1, img2, best_log
    #update_params()
    with torch.no_grad():
        cfm = CustomLocalFeatureMatcher()
        detections = cfm.detectAndMatch(timg1, timg2,
                                 det_selector.value,
                                 ori_selector.value,
                                 affine_selector.value,
                                 descriptor_selector.value,
                                 matching_selector.value,
                                 ransac_selector.value)
    show_matches()


 @disable_buttons
 def switch_images_show(_=None):
    """ Forget the current configuration and swap images. """
    global img1, img2, timg1, timg2, detections
    timg1, timg2 = timg2, timg1
    img1, img2 = img2, img1
    old_d = detections
    detections = Detections(detections.kps2,
                            detections.kps1,
                            detections.tentative_matches,
                            detections.pts_matches,
                             detections.H,
                             detections.inlier_mask,
                            detections.lafs1,
                            detections.lafs2)
    reset_H_keypoints_show()
 ```

 %% Cell type:code id: tags:

 ``` python
 alg_buttons[0].on_click(run_detector_show)
 alg_buttons[1].on_click(ransac_fit_show)
 visu_buttons[0].on_click(inliers_display_mode)
 visu_buttons[1].on_click(outliers_display_mode)


 layout_space = Layout(display="flex", justify_content="space-between")

 output_img = wd.Output()
 output_legend = wd.Output()
 output_best_legend = wd.Output()
 hb_control_panel = HBox([output_legend,
                         output_best_legend,
                         VBox(visu_buttons), VBox(alg_buttons), VBox(dropdowns)],
                        layout=layout_space)
 app = VBox([hb_control_panel, output_img])
 ```

 %% Cell type:code id: tags:

 ``` python
 ```

 %% Cell type:code id: tags:

 ``` python
 run_detector_show()
 display(app)
 ```

 %% Output

    /opt/homebrew/Caskroom/miniforge/base/envs/python39/lib/python3.9/site-packages/torch/functional.py:445: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at  ../aten/src/ATen/native/TensorShape.cpp:2157.)
      return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]