Unverified Commit 544323a6 authored by Vladyslav Yazykov's avatar Vladyslav Yazykov Committed by yazykov.v
Browse files

Added spatial verification. Set minimum score to 0

parent d74de7a6
......@@ -19,13 +19,13 @@ datasets = read_yaml("/local/vrg3_demo/vrg3_demo/app/engine_vdvm22/config.yml").
for dataset_name, dataset_config in datasets.items():
for engine_name, engine_config in dataset_config.items():
engine_data = load_data(
invfile_path=engine_config["inverted_file"],
cached_lengths_path=engine_config["cached_lengths"],
geometries_path=engine_config["geometries"],
cid2id_path=engine_config["cid2id"],
id2cid_path=engine_config["id2cid"],
image_sizes_path=engine_config["image_sizes"],
options={} if "options" not in engine_config else engine_config["options"],
invfile_path=engine_config.inverted_file,
cached_lengths_path=engine_config.cached_lengths,
geometries_path=engine_config.geometries,
cid2id_path=engine_config.cid2id,
id2cid_path=engine_config.id2cid,
image_sizes_path=engine_config.image_sizes,
options={} if "options" not in engine_config else engine_config.options,
)
engines[dataset_name][engine_name] = Engine(engine_data), engine_config
......@@ -62,7 +62,6 @@ def capabilities():
],
"images_overlays": [
{"id": "shear_bbxs", "label": "Found bbx", "type": "paths", "default": True},
# {"id": "rectangles_over", "label": "Rectangles overlay", "type": "rectangles", "default": True},
{"id": "text_over", "label": "Text overlay", "type": "text", "default": True},
{"id": "rank", "label": "Rank", "type": "text", "default": True},
{"id": "name", "label": "Name", "type": "text", "default": True},
......@@ -101,7 +100,6 @@ def images():
# Show images in a random order
mode = "browsing"
bboxes = [[]] * len(imids)
bboxes_path = [[]] * len(imids)
elif req.query.type == "image":
# This will be either [] or in the following format [{'x1': 0.2608, 'x2': 0.6889, 'y1': 0.3993, 'y2': 0.6515}]
......@@ -134,22 +132,7 @@ def images():
imids = imids[1:]
bboxes = bboxes[1:]
# TODO: uncomment for new bbox format & add the last point to close the shape
"""
bboxes_path = []
for i, bbox in enumerate(bboxes):
bboxes_path.append({"points_xy": bbox.corner_points.tolist()})
"""
bboxes_path = []
for i, bbox in enumerate(bboxes):
bboxes_path.append([{"x": bbox.x1, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y2},
{"x": bbox.x1, "y": bbox.y2},
{"x": bbox.x1, "y": bbox.y1}])
bboxes = [{"points_xy": bbox.corner_points.tolist()} for bbox in bboxes]
else:
raise ValueError("Unknown query type")
......@@ -162,29 +145,29 @@ def images():
out_data = {
"results": len(imids),
"images" : [{"prefix" : req['dataset_name'],
"path" : os.path.join(config["images_base_path"], cid2filename(engine.id2cid[rank])),
"images" : [{"prefix" : req.dataset_name,
"path" : os.path.join(config.images_base_path, cid2filename(engine.id2cid[rank])),
"overlays": {
"rank" : req['offset'] + i + 1,
"rank" : req.offset + i + 1,
"name" : engine.id2cid[rank],
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=100),
"keypoints": visualize_local_features(rank, engine, mode='point'),
"shear_bbxs" : [bboxes_path[i]],
"keypoints" : visualize_local_features(rank, engine, mode='point'),
"shear_bbxs" : [bboxes[i]],
}} for i, rank in enumerate(imids)],
}
if mode == "image":
rectangles_over = []
if req["query"]["search_mode"].get("tool") == "rectangle":
rectangles_over.append(req["query"]["search_mode"]["tool_data"])
if req.query.search_mode.get("tool") == "rectangle":
rectangles_over.append(req.query.search_mode.tool_data)
out_data["query_text"] = f"{len(imids)} results in {time.time() - time0:.3f}s for bbox: {str(rectangles_over)}"
out_data["query_image"] = {
"overlays": {
"rectangles_over": rectangles_over,
"text_over" : "search_mode=" + req['query']['search_mode']['id'],
"name" : parse_name(req["query"]["value"]["path"]),
"text_over" : "search_mode=" + req.query.search_mode.id,
"name" : parse_name(req.query.value.path),
}
}
......
......@@ -13,7 +13,7 @@ datasets:
max_MxN: 10
max_spatial: 50
inlier_threshold: 8
minimum_score: 0.1
minimum_score: 0.0
use_query_expansion: True
max_qe_keypoints: 1500
min_inliers_for_qe: 10
......@@ -111,13 +111,18 @@ class Engine:
labels = labels[mask]
positions = positions[mask]
unique_labels, unique_indices, unique_counts = np.unique(labels, return_index=True, return_counts=True)
tfidf = self.idf[unique_labels].squeeze() * unique_counts
tfidf = tfidf / np.linalg.norm(tfidf)
tfidf, unique_labels, unique_indices, unique_counts = self.get_idf(labels)
return Geometry(labels, unique_labels, positions, positions[unique_indices], tfidf)
def get_idf(self, labels: np.ndarray):
unique_labels, unique_indices, unique_counts = np.unique(labels, return_index=True, return_counts=True)
tfidf = self.idf[unique_labels].squeeze() * unique_counts
tfidf = tfidf / np.linalg.norm(tfidf)
return tfidf, unique_labels, unique_indices, unique_counts
def get_vw_in_image(self, imid: int) -> np.ndarray:
"""
Returns the visual words in the given image.
......@@ -127,9 +132,11 @@ class Engine:
with h5py.File(self.geometries_path, "r") as f:
return f[f"geom/{imid}/labels"][()].squeeze()
def query(self, visual_words, tfidf, top_k=100):
def query(self, visual_words, top_k=100):
tfidf, unique_vw, *_ = self.get_idf(visual_words)
scores = np.zeros(self.n_documents)
for vw, tfidf_vw in zip(visual_words, tfidf):
for vw, tfidf_vw in zip(unique_vw, tfidf):
offset, wf, max_bit = self.invfile_head[vw]
weight = self.idf[vw] * tfidf_vw
......
"""This file implements unconstrained search with precomputed local features."""
from typing import List, Tuple
import numpy as np
......@@ -7,7 +8,7 @@ from utilities import BBox
from .utils import make_query
def search_methods(imid: int, engine: Engine, bbox: BBox = None, mode="Unconstrained"):
def search_methods(imid: int, engine: Engine, bbox: BBox = None, mode="Unconstrained") -> Tuple[np.ndarray, List[BBox]]:
if bbox is None:
bbox = BBox()
......
......@@ -137,86 +137,73 @@ def get_bbox_data(query_vw, query_geometry, bbox: BBox):
def query_spatial_verification(imid: int, bbox: BBox, engine: Engine):
"""
:param query_visual_words: [num_keypoints_in_query x 1]
:param query_geometry: query geometry of shape [num_keypoints x 6]
:param imid: Image id
:param bbox: list [x, y, xx, yy]
:param visual_words: [num_imgs x num_words_in_img] visual_words
:param geometries: np.ndarray of object of len num_images. each object is
np.ndarray [num_keypoints_in_img x 6]
:param db: [num_words, num_imgs]
:param idf: Inverse Document Frequency. Shape: [num_words, 1]
:param params: dictionary, important keys here: minimum_score, max_tc,
max_MxN, use_query_expansion
:return:
:param engine: Engine
"""
inliers_counts, idxs, As = get_query(imid, bbox, engine)
geometry = engine.get_geometries(imid, bbox)
return inliers_counts, idxs, As
query_visual_words = geometry.labels
query_geometry = geometry.positions
inliers_counts, idxs, As = get_query(query_visual_words, query_geometry, engine)
# Query expansion will be used if the parameter use_query_expansion is
# True. The other parameters are the same.
# if params['use_query_expansion']:
# max_qe_keypoints = params['max_qe_keypoints']
# enough_inl_idx = params["min_inliers_for_qe"] < inliers_counts
#
# import sys
# print("enough_inl_idx", enough_inl_idx, file=sys.stderr)
#
# As = As[enough_inl_idx]
# idxs = idxs[enough_inl_idx]
#
# for i, id in enumerate(idxs):
# A = As[i]
# q_visual_words = visual_words[id]
# q_geometries = geometries[id]
#
# q_xs = np.hstack(
# (q_geometries[:, :2], np.ones((q_geometries.shape[0], 1)))).T
# q_xs = (np.linalg.inv(A) @ q_xs).T
# q_geometries[:, :2] = q_xs[:, :2]
#
# q_visual_words, q_geometries = get_bbox_data(q_visual_words, q_geometries,
# bbox)
# unique_vw, counts = np.unique(q_visual_words, return_counts=True)
#
# # We will skip visual words with many (more than 100) occurrences in a
# # particular database image; the associated local features should not
# # be included in the expanded query.
# keep = unique_vw[counts < 100]
# keep_mask = np.isin(q_visual_words, keep)
# q_visual_words = q_visual_words[keep_mask]
# q_geometries = q_geometries[keep_mask]
#
# query_visual_words = np.concatenate((query_visual_words, q_visual_words))
# query_geometry = np.concatenate((query_geometry, q_geometries))
#
# # We will restrict the number of local features in the expanded query
# # to be less than max_qe_keypoints, simply by skipping the rest of the
# # spatially verified images.
# if query_visual_words.shape[0] >= max_qe_keypoints:
# query_geometry = query_geometry[:max_qe_keypoints]
# query_visual_words = query_visual_words[:max_qe_keypoints]
# break
#
# inliers_counts, idxs, As = get_query(query_visual_words, visual_words,
# db, idf, params, query_geometry,
# geometries)
# return inliers_counts, idxs, As
def get_query(imid: int, bbox: BBox, engine: Engine):
geometry = engine.get_geometries(imid, bbox)
if engine.options["use_query_expansion"]:
max_qe_keypoints = engine.options["max_qe_keypoints"]
enough_inl_idx = engine.options["min_inliers_for_qe"] < inliers_counts
As = As[enough_inl_idx]
idxs = idxs[enough_inl_idx]
for i, id in enumerate(idxs):
A = As[i]
g = engine.get_geometries(id)
q_visual_words = g.labels
q_geometries = g.positions
q_xs = np.hstack((q_geometries[:, :2], np.ones((q_geometries.shape[0], 1)))).T
q_xs = (np.linalg.inv(A) @ q_xs).T
q_geometries[:, :2] = q_xs[:, :2]
q_visual_words, q_geometries = get_bbox_data(q_visual_words, q_geometries, bbox)
unique_vw, counts = np.unique(q_visual_words, return_counts=True)
# We will skip visual words with many (more than 100) occurrences in a
# particular database image; the associated local features should not
# be included in the expanded query.
keep = unique_vw[counts < 100]
keep_mask = np.isin(q_visual_words, keep)
q_visual_words = q_visual_words[keep_mask]
q_geometries = q_geometries[keep_mask]
query_visual_words = np.concatenate((query_visual_words, q_visual_words))
query_geometry = np.concatenate((query_geometry, q_geometries))
# We will restrict the number of local features in the expanded query
# to be less than max_qe_keypoints, simply by skipping the rest of the
# spatially verified images.
if query_visual_words.shape[0] >= max_qe_keypoints:
query_geometry = query_geometry[:max_qe_keypoints]
query_visual_words = query_visual_words[:max_qe_keypoints]
break
inliers_counts, idxs, As = get_query(query_visual_words, query_geometry, engine)
return inliers_counts, idxs, As
idxs, scores = engine.query(geometry.unique_labels, geometry.tfidf)
def get_query(labels, positions, engine: Engine) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
idxs, scores = engine.query(labels)
idxs = idxs[scores > engine.options['minimum_score']]
idxs = idxs[: min(len(idxs), engine.options['max_spatial'])]
corresp = get_tentative_correspondences(geometry.labels, idxs, engine)
corresp = get_tentative_correspondences(labels, idxs, engine)
As, inliers_counts = ransac_affine(geometry.positions, engine, corresp, idxs, engine.options.inlier_threshold)
As, inliers_counts = ransac_affine(positions, engine, corresp, idxs, engine.options.inlier_threshold)
sort_idxs = np.argsort(inliers_counts)[::-1]
inliers_counts = inliers_counts[sort_idxs]
......
import yaml
from .dotdict import dotdict
......@@ -7,4 +8,4 @@ def read_yaml(file_name: str) -> dotdict:
Reads a YAML file and returns a dictionary.
"""
with open(file_name, "r") as f:
return dotdict(yaml.safe_load(f))
return dotdict.from_dict(yaml.safe_load(f))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment