Commit 1041e221 authored by Vladyslav Yazykov's avatar Vladyslav Yazykov
Browse files

Fixes to draw bbox, keypoints, and ellipses

parent d5988b62
......@@ -66,7 +66,8 @@ def capabilities():
{"id": "text_over", "label": "Text overlay", "type": "text", "default": True},
{"id": "rank", "label": "Rank", "type": "text", "default": True},
{"id": "name", "label": "Name", "type": "text", "default": True},
{"id": "loc_features", "label": "Local features", "type": "ellipses", "default": True},
{"id": "keypoints", "label": "Keypoints", "type": "ellipses", "default": False},
{"id": "loc_features", "label": "Local features", "type": "ellipses", "default": False},
],
"engine_options" : [
{"id" : "mode", "label": "Search mode",
......@@ -100,6 +101,7 @@ def images():
# Show images in a random order
mode = "browsing"
bboxes = [[]] * len(imids)
bboxes_path = [[]] * len(imids)
elif req.query.type == "image":
# This will be either [] or in the following format [{'x1': 0.2608, 'x2': 0.6889, 'y1': 0.3993, 'y2': 0.6515}]
......@@ -124,6 +126,7 @@ def images():
search_mode = req.engine_options.mode
# bboxes in format out['paths_over'] = [{"points_xy": [[0.4, 0.5], [0.45, 0.6], [0.6, 0.4]]}
imids, bboxes = search_methods(imid, engine, bbox=user_bbox, mode=search_mode)
# Remove query from results
......@@ -131,14 +134,22 @@ def images():
imids = imids[1:]
bboxes = bboxes[1:]
# TODO: uncomment for new bbox format & add the last point to close the shape
"""
bboxes_path = []
for i, bbox in enumerate(bboxes):
bboxes[i] = [
{"x": bbox.x1, "y": bbox.y1},
bboxes_path.append({"points_xy": bbox.corner_points.tolist()})
"""
bboxes_path = []
for i, bbox in enumerate(bboxes):
bboxes_path.append([{"x": bbox.x1, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y2},
{"x": bbox.x2, "y": bbox.y1},
{"x": bbox.x1, "y": bbox.y1},
]
{"x": bbox.x1, "y": bbox.y2},
{"x": bbox.x1, "y": bbox.y1}])
else:
raise ValueError("Unknown query type")
......@@ -156,9 +167,9 @@ def images():
"overlays": {
"rank" : req['offset'] + i + 1,
"name" : engine.id2cid[rank],
"loc_features": visualize_local_features(rank, engine, mode='point', n=100),
# mode = 'full_geom'; 'point'
"shear_bbxs" : [bboxes[i]],
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=100),
"keypoints": visualize_local_features(rank, engine, mode='point'),
"shear_bbxs" : [bboxes_path[i]],
}} for i, rank in enumerate(imids)],
}
......
from typing import Optional
from image_search.image_search import *
from image_search.spatial_verification import *
import scipy.io
import numpy as np
import os
import pickle
import imagesize
import warnings
warnings.filterwarnings("ignore")
class Engine:
def __init__(self, file_path = '/local/vrg3_demo/vrg3_demo/data/bow_600k/mpv_files/mpvdb50k_haff2.mat', external_idf: Optional[np.ndarray] = None):
self.num_words = 50001
self.data = scipy.io.loadmat(file_path)
self.visual_words = np.array([x[0] for x in self.data['VW'][0]], dtype=object)
self.img_names = np.array([x[0][0] + ".jpg" for x in self.data['NAMES']], dtype=object)
# information about each point is in a form of [x, y, a11, a12, a21, a22]
self.geometries = np.array([x.T for x in self.data['GEOM'][0]], dtype=object)
print("Loaded geometrie")
self.idf = get_idf(self.visual_words, self.num_words)
print("Computed idf")
try:
with open('/local/vrg3_demo/vrg3_demo/data/bow_600k/mpv_files/db_tfidf.pickle', 'rb') as handle:
self.db = pickle.load(handle)
print("Loaded a saved db_tfidf")
except Exception:
self.db = create_db_tfidf(self.visual_words, self.num_words, self.idf)
print("Created db")
with open('/local/vrg3_demo/vrg3_demo/data/bow_600k/mpv_files/db_tfidf.pickle', 'wb') as handle:
pickle.dump(self.db, handle, protocol=pickle.HIGHEST_PROTOCOL)
self.options = {
'max_tc' : 600,
'max_MxN' : 10,
'max_spatial' : 50,
'inlier_threshold' : 8,
'data_root_dir' : 'mpv_images/',
'data_root_full_dir' : '/local/vrg3_demo/vrg3_demo/data/bow_600k/',
'minimum_score' : 0.1,
'use_query_expansion': True,
'max_qe_keypoints' : 1500,
'min_inliers_for_qe' : 10
}
self.paths = [os.path.join(self.options["data_root_dir"], i_name) for i_name in self.img_names]
def get_img_size(self, img_path_rel):
# returns [width, height]
img_path_abs = os.path.join(self.options["data_root_full_dir"], img_path_rel)
return imagesize.get(img_path_abs)
from typing import Optional
import os
import numpy as np
import pickle
from utilities import dotdict, BinaryFile, sizeof_uint64_t, sizeof_unsigned_char, sizeof_unsigned_int, cid2filename
class Engine:
"""Corresponds to the engine3 in cpp files (/home.dokt/jenicto2/cmpg2_demo/wbs/mexsrc/engine/en3_loadup.mex.cpp)
-> (/home.dokt/jenicto2/cmpg2_demo/wbs/mexsrc/engine/en3_get.mex.cpp )"""
def __init__(self, invfile_path: str, external_idf: Optional[np.ndarray] = None):
"""
Initializes the engine.
1. Reads the invfile from the path provided
2. If external idf is not provided, calculates the idf from the database
:param invfile_path: Path to the invfile.dat
:param external_idf: Shape: [num_labels, 1] - if provided, will use this idf instead of calculating it from the database
"""
# Loading cid->id and id->cid conversion files
base_data_loading_path = "/local/vrg3_demo/vrg3_demo/app/engine_vdvm22/data_loading/"
with open(os.path.join(base_data_loading_path, 'cid2id.pkl'), 'rb') as f:
self.cid2id = pickle.load(f)
with open(os.path.join(base_data_loading_path, 'id2cid.pkl'), 'rb') as f:
self.id2cid = pickle.load(f)
# Creating paths to all of the images, corresponding ids and cids
self.paths = [cid2filename(cid) for cid in self.cid2id.keys()]
self.ids = list(self.cid2id.values())
self.cids = list(self.cid2id.keys())
# Reading the inverted file
with BinaryFile(invfile_path) as f:
self.engine_version = f.read_string(8)
self.num_clusters = f.read_int() # number of clusters
self.num_documents = f.read_int() # todo: what is this?
# Reading actual inverted file matrix: visual words
self.data_length_in_bytes = f.read_uint64()
self.data = f.read_bytes(self.data_length_in_bytes * sizeof_uint64_t)
sizeof_ifile_head = sizeof_uint64_t + sizeof_unsigned_int + sizeof_unsigned_char
ifile_head_bytes = f.read_bytes(self.num_clusters * sizeof_ifile_head)
self.ifile_head = np.frombuffer(ifile_head_bytes, dtype=(np.dtype([('offset', 'Q'), ('wf', 'I'), ('maxbit', 'B')])))
self.code_bits = f.read_list(4, "int") # todo: what is this?
if external_idf is not None:
self.idf: np.ndarray = external_idf
else:
# Using document frequency, in how many documents does the vw appear
document_frequencies = f.read_list(self.num_clusters, "unsigned")
document_frequencies = np.array(document_frequencies)
self.idf: np.ndarray = np.log(self.num_documents / document_frequencies)
self.idf[np.isinf(self.idf)] = 0
# This is basically an inlined answer from the database call
self.opt = dotdict({
"dataset" : "bigzoom",
"geometry_cfg": {
"scale_bits" : 4,
"ellipse_bits" : 12,
"bits" : [11, 13, 15],
"inverted_compression_bits": [11, 12, 14],
},
"git_tag" : "f1.1",
"version" : 4,
"vhash" : "655e52775a53eabff09ba46e8055eb15",
})
# commands = ["idf", "geom", "size", "scalebins", "invertedlist", "docnfeat", "tcorr"]
# en3_get
def get_idf(self, return_word_frequencies=False):
idf = self.idf.copy() # todo: is it necessary to make a copy?
if not return_word_frequencies:
return idf
return idf, self.get_word_frequencies()
def get_word_frequencies(self):
return self.ifile_head["wf"]
def load_invfiles(invfiles):
loaded = {}
for dataset, engines in invfiles.items():
loaded[dataset] = {}
for engine, invfile_path in engines.items():
loaded[dataset][engine] = Engine(invfile_path)
return loaded
if __name__ == '__main__':
invfile_path = "/local/vrg3_demo/vrg3_demo/data/bow_600k/files/invfile.dat"
engine = Engine(invfile_path)
print(f"Engine version: {engine.engine_version}")
print(f"Clus {engine.num_clusters}")
print(f"Doc {engine.num_documents}")
print(f"Dlen {engine.data_length_in_bytes}")
print(f"Data read {len(engine.data)}")
print("Idf")
# TODO
File mode changed from 100644 to 100755
......@@ -11,22 +11,20 @@ def make_query(imid: int, bbox: BBox, engine: Engine):
bboxes = []
for img_id, inl, A in zip(idxs, scores, As):
width, height = engine.image_sizes[img_id]
height, width = engine.image_sizes[img_id]
corner_pts = np.array([
[bbox.x1, bbox.y1, 1],
[bbox.x2, bbox.y2, 1],
])
[bbox.x1, bbox.y1, 1],
[bbox.x2, bbox.y2, 1],
], dtype=np.int)
corner_pts = corner_pts @ A.T
bbox = BBox(
x1=corner_pts[0, 0] / width,
y1=corner_pts[0, 1] / height,
x2=corner_pts[1, 0] / width,
y2=corner_pts[1, 1] / height,
box = BBox(
x1=corner_pts[0, 0].astype(np.int) / width,
y1=corner_pts[0, 1].astype(np.int) / height,
x2=corner_pts[1, 0].astype(np.int) / width,
y2=corner_pts[1, 1].astype(np.int) / height,
)
bboxes.append(bbox)
bboxes.append(box)
return idxs, bboxes
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
%% Cell type:code id: tags:
``` python
#@formatter:off
%load_ext autoreload
%autoreload 2
#@formatter:on
```
%% Cell type:markdown id: tags:
`en3_loadup.mex.cpp`
It basically creates en engine from the inverted file document:
%% Cell type:code id: tags:
``` python
from loading import load_data
from utilities import read_yaml
engine_config = read_yaml("../config.yml").datasets["bow_600k"]["zoom_in"]
engine_data = load_data(
invfile_path="../../../data/invfile.dat",
cached_lengths_path="../../../data/cached_lengths.dat",
geometries_path="../../../data/geometries.h5",
cid2id_path="../../../data/cid2id.pkl",
id2cid_path="../../../data/id2cid.pkl",
image_sizes_path="../../../data/image_sizes.npy",
options={} if "options" not in engine_config else engine_config["options"],
)
data = load_data("../../data/correct_invfile.dat", "../../data/cid2id.pkl", "../../data/id2cid.pkl")
print("Data loaded")
```
%% Output
C:\dev\university\PVTY\vdvm\engine_vdvm22\loading\load_data.py:31: RuntimeWarning: divide by zero encountered in true_divide
idf = np.log(num_documents / document_frequencies)
Data loaded
%% Cell type:code id: tags:
``` python
from engine import Engine
engine = Engine(data)
engine = Engine(engine_data)
print("Engine created")
```
%% Output
Engine created
%% Cell type:code id: tags:
``` python
# Check that our code is equivalent to the old one
print(f"Engine version: {engine.version}")
print(f"Clus {engine.n_vw}")
print(f"Doc {engine.n_documents}")
print(f"Dlen {int(len(engine.invfile_bytes) / 8)}")
print(f"Data read {len(engine.invfile_bytes)}")
print(f"Ifile read {len(engine.invfile_head) * 13}")
print(f"Code bits {engine.code_bits}")
print(f"Documents: {engine.n_documents}")
```
%% Output
Engine version: ENG v3.0
Clus 16777216
Doc 619026
Dlen 292298207
Data read 2338385656
Dlen 293826684
Data read 2350613472
Ifile read 218103808
Code bits [11, 12, 14, 5]
Code bits (11, 12, 14)
Documents: 619026
%% Cell type:markdown id: tags:
`en3_get.mex.cpp`
gets some properties of the engine:
%% Cell type:code id: tags:
``` python
import numpy as np
np.set_printoptions(precision=4)
print(f"{'idf:':>20}", engine.idf)
print(f"{'word_frequencies:':>20}", engine.invfile_head.wf)
vw = engine.invfile_head[0]
print(f"{'vw[0].header:':>20} (offset={vw.offset}, word_frequency={vw.wf}, max_bit={vw.max_bit})")
```
%% Output
idf: [ 9.9019 8.5076 8.2545 ... 7.7264 8.2001 11.5441]
word_frequencies: [ 32 127 165 ... 274 173 6]
vw[0].header: (offset=0, word_frequency=32, max_bit=16)
%% Cell type:markdown id: tags:
We can get the documents that contain a visual word like this:
%% Cell type:code id: tags:
``` python
vw = 123
documents_with_vw = engine.get_documents_with_vw(vw)
print(f"Documents with vw {vw}: {documents_with_vw}")
```
%% Output
Documents with vw 123: [ 16805 17997 20271 21666 36587 45406 46261 47960 57277 63681 71496 94024 101019 116480 119064 120314 124386 124678 133215 135231 139026 143778 145126 167654 190182 212710 215608 238136 242333 253810 254701 277229 299757 322285 325915 327971 328254 328676 338524 338951 361479 368705 385453 388253 393253 394749 417277 419028 419972 421347 424021 425294 427700 428838 433362 433362 433997 438117 460645 462194 471732 494260 494635 500555 518648 541176 546781 547867 560346 563532 586060 586579 587835 590973 607828 630356 632356 654884 672039 679500 680566 686345 708873 709051 710598 712407 733167 733167 733167 755695 758064 761043 766035 766035 766163 766163 766163 766216 768258 790786 813314 835842 855960 856984 856994 879522 888001 910529 931553]
......
import numpy as np
from numpy.random import RandomState
from engine import Engine
def get_elipse_from_geom(geom: np.ndarray, h, w):
x = geom[0]
y = geom[1]
a = geom[2] / w
b = geom[3] / h
c = geom[4] / h
A = np.array([[a, 0],
[b, c]])
eig, V = np.linalg.eig(A)
return {"x": x / w, "y": y / h, "a": eig[0], "b": eig[1], "rotation": np.arctan2(V[0][1], V[0][0])}
def get_point_from_geom(geom: np.ndarray, h, w):
return {"x": geom[0] / w, "y": geom[1] / h, "a": 0.005, "b": 0.005, "rotation": 0}
def visualize_local_features(cid, engine: Engine, mode='full_geom', n=100):
imid = engine.cid2id[cid]
geometry = engine.get_geometries(imid)
geometry = geometry.positions
w, h = engine.image_sizes[imid]
ellipses = []
if mode == 'full_geom':
rand = RandomState(1234567890)
for geom in geometry[rand.choice(geometry.shape[0], n)]:
ellipses.append(get_elipse_from_geom(geom, h, w))
if mode == 'point':
for geom in geometry:
ellipses.append(get_point_from_geom(geom, h, w))
return ellipses
......@@ -5,40 +5,38 @@ from engine import Engine
def get_ellipse_from_geom(geom: np.ndarray, h, w):
x = geom[0]
y = geom[1]
a = geom[2] / w
b = geom[3] / h
c = geom[4] / h
a = geom[2]
b = geom[3]
c = geom[4]
A = np.array([[a, 0],
[b, c]])
eig, V = np.linalg.eig(A)
return {"x": x / w, "y": y / h, "a": eig[0], "b": eig[1], "rotation": np.arctan2(V[0][1], V[0][0])}
return {"x": geom[0] / w, "y": geom[1] / h, "a": eig[0], "b": eig[1], "rotation": np.arctan2(V[0][1], V[0][0])}
def get_point_from_geom(geom: np.ndarray, h, w):
return {"x": geom[0] / w, "y": geom[1] / h, "a": 0.005, "b": 0.005, "rotation": 0}
return {"x": geom[0] / w, "y": geom[1] / h, "a": 0.001, "b": 0.001, "rotation": 0, "color": "#ffff00"}
def visualize_local_features(imid: int, engine: Engine, mode='full_geom', n=100):
geometry = engine.get_geometries(imid)
geometry = geometry.positions
w, h = engine.image_sizes[imid]
h, w = engine.image_sizes[imid]
ellipses = []
if mode == 'full_geom':
rand = RandomState(1234567890)
for geom in geometry[rand.choice(geometry.shape[0], n)]:
rand = RandomState(0)
num_features = geometry.shape[0]
for geom in geometry[rand.choice(num_features, min(n, num_features))]:
ellipses.append(get_ellipse_from_geom(geom, h, w))
if mode == 'point':
for geom in geometry:
ellipses.append(get_point_from_geom(geom, h, w))
#raise ValueError(ellipses)
return ellipses
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment