Commit ddc22dbd authored by Vladyslav Yazykov's avatar Vladyslav Yazykov
Browse files

Zoom-in/out are kinda working

parent 1e488362
......@@ -65,7 +65,9 @@ def capabilities():
{"id": "text_over", "label": "Text overlay", "type": "text", "default": True},
{"id": "rank", "label": "Rank", "type": "text", "default": True},
{"id": "name", "label": "Name", "type": "text", "default": True},
{"id": "num_inliers", "label": "Inliers", "type": "text", "default": True},
{"id": "keypoints", "label": "Keypoints", "type": "ellipses", "default": False},
{"id": "matches", "label": "Matches", "type": "ellipses", "default": False},
{"id": "loc_features", "label": "Local features", "type": "ellipses", "default": False},
],
"engine_options": [
......@@ -101,6 +103,8 @@ def images():
mode = "browsing"
bboxes = [BBox()] * len(imids)
bboxes_path = [[]] * len(imids)
num_inliers = [[]] * len(imids)
score = [[]] * len(imids)
user_bbox = BBox()
imid = None
......@@ -129,7 +133,7 @@ def images():
search_mode = req.engine_options.mode
# bboxes in format out['paths_over'] = [{"points_xy": [[0.4, 0.5], [0.45, 0.6], [0.6, 0.4]]}
imids, bboxes = search_methods(imid, engine, bbox=user_bbox_mul, mode=search_mode)
imids, bboxes, num_inliers, score = search_methods(imid, engine, bbox=user_bbox_mul, mode=search_mode)
# Remove query from results
if search_mode == "Unconstrained":
......@@ -163,9 +167,12 @@ def images():
"overlays": {
"rank": req.offset + i + 1,
"name": engine.id2cid[rank],
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=100, bbox=bboxes[i], query=imid,
"num_inliers": "Inl: " + str(num_inliers[i]) + " Sc: " + str(score[i]),
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=100, query=imid,
user_bbox=user_bbox),
"keypoints": visualize_local_features(rank, engine, mode='point', bbox=bboxes[i]),
"matches": visualize_local_features(rank, engine, mode='full_geom', n=100, query=imid,
user_bbox=user_bbox),
"shear_bbxs": [bboxes_path[i]],
}} for i, rank in enumerate(imids)],
}
......
......@@ -9,11 +9,11 @@ datasets:
image_sizes: /local/vrg3_demo/vrg3_demo/data/bow_600k/files/image_sizes.npy
images_base_path: images/
options:
max_tc: 600
max_tc: 1000
max_MxN: 10
max_spatial: 50
max_spatial: 70
inlier_threshold: 8
minimum_score: 0.0
minimum_score: 0.01
use_query_expansion: True
max_qe_keypoints: 1500
min_inliers_for_qe: 10
min_inliers_for_qe: 20
......@@ -62,6 +62,8 @@ class Engine:
self.visual_words = LambdaDict(lambda vw: self.get_documents_with_vw(vw))
self.geometries = LambdaDict(lambda imid: self.get_geometries(imid)[1])
self.geom_file = h5py.File(self.geometries_path, "r")
def image_paths(self, size=100, random=True):
size = min(size, self.n_documents)
......@@ -93,27 +95,26 @@ class Engine:
:return: [labels, geometries, tfidf] where labels are unique visual words in the image,
geometries is an array of corresponding geometries, and tfidf is tf-idf weights of the visual words.
"""
with h5py.File(self.geometries_path, "r") as f:
labels = f[f"geom/{imid}/labels"][()].squeeze()
positions = f[f"geom/{imid}/pos"][()]
labels = self.geom_file[f"geom/{imid}/labels"][()].squeeze()
positions = self.geom_file[f"geom/{imid}/pos"][()]
# inverting of the transformation matrix of the ellipses from the initial de-normalizing to the normalizing one
positions[:, 2] = 1. / positions[:, 2]
positions[:, 4] = 1. / positions[:, 4]
positions[:, 3] = - positions[:, 3] * positions[:, 2] * positions[:, 4]
# inverting of the transformation matrix of the ellipses from the initial de-normalizing to the normalizing one
positions[:, 2] = 1. / positions[:, 2]
positions[:, 4] = 1. / positions[:, 4]
positions[:, 3] = - positions[:, 3] * positions[:, 2] * positions[:, 4]
if bbox is not None:
x = positions[:, 0]
y = positions[:, 1]
if bbox is not None:
x = positions[:, 0]
y = positions[:, 1]
mask = (x > bbox.x1) & (y > bbox.y1) & (x < bbox.x2) & (y < bbox.y2)
mask = (x > bbox.x1) & (y > bbox.y1) & (x < bbox.x2) & (y < bbox.y2)
labels = labels[mask]
positions = positions[mask]
labels = labels[mask]
positions = positions[mask]
tfidf, unique_labels, unique_indices, unique_counts = self.get_idf(labels)
tfidf, unique_labels, unique_indices, unique_counts = self.get_idf(labels)
return Geometry(labels, unique_labels, positions, positions[unique_indices], tfidf)
return Geometry(labels, unique_labels, positions, positions[unique_indices], tfidf)
def get_idf(self, labels: np.ndarray):
unique_labels, unique_indices, unique_counts = np.unique(labels, return_index=True, return_counts=True)
......
......@@ -5,21 +5,21 @@ from utilities import BBox
from .spatial_verification import query_spatial_verification
def make_query(imid: int, bbox: BBox, engine: Engine):
scores, idxs, As = query_spatial_verification(imid, bbox, engine)
def make_query(imid: int, bbox: BBox, engine: Engine, verify: bool):
inliers_counts, idxs, As, score = query_spatial_verification(imid, bbox, engine, verify)
bboxes = []
for img_id, inl, A in zip(idxs, scores, As):
for img_id, A in zip(idxs, As):
height, width = engine.image_sizes[img_id]
corner_pts = np.array([
[bbox.x1, bbox.y1, 1],
[bbox.x2, bbox.y2, 1],
[bbox.x1, bbox.y1, 1],
[bbox.x2, bbox.y2, 1],
], dtype=np.int)
corner_pts = corner_pts @ A.T
box = BBox(
x1=corner_pts[0, 0].astype(np.int) / width,
y1=corner_pts[0, 1].astype(np.int) / height,
......@@ -27,4 +27,4 @@ def make_query(imid: int, bbox: BBox, engine: Engine):
y2=corner_pts[1, 1].astype(np.int) / height,
)
bboxes.append(box)
return idxs, bboxes
return idxs, bboxes, inliers_counts, score
......@@ -8,28 +8,33 @@ from utilities import BBox
from .make_query import make_query
def search_methods(imid: int, engine: Engine, bbox: BBox = None, mode="Unconstrained") -> Tuple[np.ndarray, List[BBox]]:
def search_methods(imid: int, engine: Engine, bbox: BBox, mode="Unconstrained") -> Tuple[np.ndarray, List[BBox], np.ndarray]:
if bbox is None:
bbox = BBox()
# bboxes are in [0, 1] range
ranks, bboxes = make_query(imid, bbox, engine)
verify = mode != "Unconstrained" # Require spatial verification for zoom-in and zoom-out
ranks, bboxes, num_inliers, score = make_query(imid, bbox, engine, verify)
if mode == "Zoom-in":
scores = [bb.area for r, bb in zip(ranks, bboxes)]
scores = [bb.visible_area for r, bb in zip(ranks, bboxes)]
sort = np.argsort(scores)[::-1]
ranks = [ranks[i] for i in sort]
bboxes = [bboxes[i] for i in sort]
num_inliers = [num_inliers[i] for i in sort]
score = [score[i] for i in sort]
elif mode == "Zoom-out":
scores = [bb.area for r, bb in zip(ranks, bboxes)]
scores = [bb.area for r, bb in zip(ranks, bboxes) if bb.whole_visible]
sort = np.argsort(scores)
ranks = [ranks[i] for i in sort]
bboxes = [bboxes[i] for i in sort]
num_inliers = [num_inliers[i] for i in sort]
score = [score[i] for i in sort]
elif mode == "Unconstrained":
pass
......@@ -37,4 +42,4 @@ def search_methods(imid: int, engine: Engine, bbox: BBox = None, mode="Unconstra
else:
raise ValueError("Unknown search mode")
return ranks, bboxes
return ranks, bboxes, num_inliers, score
from typing import Tuple
from typing import List, Tuple
import numpy as np
......@@ -6,7 +6,7 @@ from engine import Engine
from utilities import BBox
def get_tentative_correspondences(lbl: np.ndarray, idxs: np.ndarray, engine: Engine) -> np.ndarray:
def get_tentative_correspondences(lbl: np.ndarray, idxs: np.ndarray, engine: Engine) -> List[np.ndarray]:
"""
Compute tentative correspondences.
......@@ -42,7 +42,7 @@ def get_tentative_correspondences(lbl: np.ndarray, idxs: np.ndarray, engine: Eng
correspondences.append(tentative_correspondences)
return np.array(correspondences)
return correspondences
def get_A_matrix_from_geom(geom: np.ndarray):
......@@ -90,6 +90,7 @@ def ransac_affine(
"""
As = []
inliers_counts = []
inliers = []
for c, rel_idx in enumerate(idxs):
D_geom = engine.get_geometries(rel_idx).positions
......@@ -102,6 +103,7 @@ def ransac_affine(
best_A = None
best_support = 0
best_inliers = []
for x, y in corresp:
# corresp to canonical coordinates
......@@ -116,11 +118,15 @@ def ransac_affine(
if support > best_support:
best_support = support
best_A = A
best_inliers = [corresp[i] for i in range(len(errors)) if errors[i] <= inlier_threshold]
assert best_A is not None, f"ransac_affine: best_A is None for image {rel_idx}"
inliers_counts.append(best_support)
As.append(best_A)
inliers.append(best_inliers)
return np.array(As), np.array(inliers_counts)
return np.array(As), np.array(inliers_counts), np.array(inliers)
def get_bbox_data(query_vw, query_geometry, bbox: BBox):
......@@ -135,7 +141,7 @@ def get_bbox_data(query_vw, query_geometry, bbox: BBox):
return bbox_vw, bbox_geom
def query_spatial_verification(imid: int, bbox: BBox, engine: Engine):
def query_spatial_verification(imid: int, bbox: BBox, engine: Engine, verify: bool):
"""
:param imid: Image id
:param bbox: list [x, y, xx, yy]
......@@ -146,7 +152,7 @@ def query_spatial_verification(imid: int, bbox: BBox, engine: Engine):
query_visual_words = geometry.labels
query_geometry = geometry.positions
inliers_counts, idxs, As = get_query(query_visual_words, query_geometry, engine)
inliers_counts, idxs, As, score = get_query(query_visual_words, query_geometry, engine)
# Query expansion will be used if the parameter use_query_expansion is
# True. The other parameters are the same.
......@@ -190,24 +196,48 @@ def query_spatial_verification(imid: int, bbox: BBox, engine: Engine):
query_visual_words = query_visual_words[:max_qe_keypoints]
break
inliers_counts, idxs, As = get_query(query_visual_words, query_geometry, engine)
inliers_counts, idxs, As, score = get_query(query_visual_words, query_geometry, engine)
if verify:
enough_inl_idx = inliers_counts > engine.options["min_inliers_for_qe"]
As = As[enough_inl_idx]
idxs = idxs[enough_inl_idx]
score = score[enough_inl_idx]
inliers_counts = inliers_counts[enough_inl_idx]
return inliers_counts, idxs, As
return inliers_counts, idxs, As, score
def get_query(labels, positions, engine: Engine) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
assert labels.shape[0] > 0, "No labels"
assert labels.shape[0] == positions.shape[0]
idxs, scores = engine.query(labels)
idxs = idxs[scores > engine.options['minimum_score']]
enough_scores_mask = scores > engine.options['minimum_score']
scores = scores[enough_scores_mask]
idxs = idxs[enough_scores_mask]
idxs = idxs[: min(len(idxs), engine.options['max_spatial'])]
corresp = get_tentative_correspondences(labels, idxs, engine)
As, inliers_counts = ransac_affine(positions, engine, corresp, idxs, engine.options.inlier_threshold)
# Filter out correspondences with length 0
_corresp, _idxs = [], []
for c, idx in zip(corresp, idxs):
if c.shape[0] > 0:
_corresp.append(c)
_idxs.append(idx)
corresp, idxs = _corresp, np.array(_idxs)
As, inliers_counts, inliers = ransac_affine(positions, engine, corresp, idxs, engine.options.inlier_threshold)
sort_idxs = np.argsort(inliers_counts)[::-1]
inliers_counts = inliers_counts[sort_idxs]
idxs = idxs[sort_idxs]
As = As[sort_idxs]
scores = scores[sort_idxs]
return inliers_counts, idxs, As
return inliers_counts, idxs, As, scores
......@@ -24,10 +24,26 @@ class BBox:
def height(self):
return self.y2 - self.y1
@property
def visible_width(self):
return min(self.x2, 1) - max(self.x1, 0)
@property
def visible_height(self):
return min(self.y2, 1) - max(self.y1, 0)
@property
def area(self):
return self.width * self.height
@property
def visible_area(self):
return self.visible_width * self.visible_height
@property
def whole_visible(self):
return self.visible_width == self.width and self.visible_height == self.height
@property
def corner_points(self):
return np.array([
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment