Commit f89ab21e authored by Vladyslav Yazykov's avatar Vladyslav Yazykov
Browse files

Unconstrained search seems to be working properly

parent f75e555a
......@@ -11,7 +11,7 @@ from werkzeug.exceptions import HTTPException
from engine import Engine
from image_search import search_methods
from loading import load_data
from utilities import Rect, cid2filename, dotdict, read_yaml
from utilities import BBox, cid2filename, dotdict, read_yaml
from visualize_local_features import visualize_local_features
engines = defaultdict(dict)
......@@ -55,8 +55,8 @@ def _parse_request(request):
@app.route("/capabilities", methods=['POST'])
def capabilities():
data = {"data": {
"search_types" : ["image"],
"search_modes" : [
"search_types": ["image"],
"search_modes": [
{"id": "search", "type": "image", "label": "Search"},
{"id": "box", "type": "image", "tool": "rectangle", "label": "Bbox"},
],
......@@ -68,9 +68,9 @@ def capabilities():
{"id": "keypoints", "label": "Keypoints", "type": "ellipses", "default": False},
{"id": "loc_features", "label": "Local features", "type": "ellipses", "default": False},
],
"engine_options" : [
{"id" : "mode", "label": "Search mode",
"type" : "option", "values": ["Unconstrained", "Zoom-in", "Zoom-out"],
"engine_options": [
{"id": "mode", "label": "Search mode",
"type": "option", "values": ["Unconstrained", "Zoom-in", "Zoom-out"],
"default": "Unconstrained"},
],
}}
......@@ -99,14 +99,14 @@ def images():
# Show images in a random order
mode = "browsing"
bboxes = [[]] * len(imids)
# Query imid
bboxes = [BBox()] * len(imids)
bboxes_path = [[]] * len(imids)
user_bbox = BBox()
imid = None
elif req.query.type == "image":
# This will be either [] or in the following format [{'x1': 0.2608, 'x2': 0.6889, 'y1': 0.3993, 'y2': 0.6515}]
# if searched using the bounding box;
mode = "image"
rectangles_over = []
if req.query.search_mode.get("tool") == "rectangle":
rectangles_over.append(req.query.search_mode.tool_data)
......@@ -118,31 +118,34 @@ def images():
# Transform user bbox to required format
if not rectangles_over:
# Set the bounding box to the full size of the image
bbox = Rect()
bbox = BBox()
else:
rect = rectangles_over[0]
bbox = Rect(rect.x1, rect.y1, rect.x2, rect.y2)
bbox = BBox(rect.x1, rect.y1, rect.x2, rect.y2)
user_bbox = bbox * engine.image_sizes[imid]
user_bbox = bbox
user_bbox_mul = bbox * engine.image_sizes[imid][::-1] # note: width and height are reversed!
search_mode = req.engine_options.mode
# bboxes in format out['paths_over'] = [{"points_xy": [[0.4, 0.5], [0.45, 0.6], [0.6, 0.4]]}
imids, bboxes = search_methods(imid, engine, bbox=user_bbox, mode=search_mode)
imids, bboxes = search_methods(imid, engine, bbox=user_bbox_mul, mode=search_mode)
# Remove query from results
if search_mode == "Unconstrained":
imids = imids[1:]
bboxes = bboxes[1:]
# todo: change the bboxes format the new front-end is ready
# bboxes = [{"points_xy": bbox + [bbox[0]]} for bbox in bboxes]
bboxes_path = []
for i, bbox in enumerate(bboxes):
bboxes_path.append([{"x": bbox.x1, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y1},
{"x": bbox.x2, "y": bbox.y2},
{"x": bbox.x1, "y": bbox.y2},
{"x": bbox.x1, "y": bbox.y1}])
bboxes = [[{"x": bbox[0][0], "y": bbox[0][1]},
{"x": bbox[1][0], "y": bbox[1][1]},
{"x": bbox[2][0], "y": bbox[2][1]},
{"x": bbox[3][0], "y": bbox[3][1]},
{"x": bbox[0][0], "y": bbox[0][1]}] for bbox in bboxes]
# todo: change later when new frontend is ready
# bboxes_path = [{"points_xy": bbox.corner_points.tolist()} for bbox in bboxes]
else:
raise ValueError("Unknown query type")
......@@ -155,15 +158,16 @@ def images():
out_data = {
"results": len(imids),
"images" : [{"prefix" : req.dataset_name,
"path" : os.path.join(config.images_base_path, cid2filename(engine.id2cid[rank])),
"overlays": {
"rank" : req.offset + i + 1,
"name" : engine.id2cid[rank],
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=50, query=imid),
"keypoints" : visualize_local_features(rank, engine, mode='point'),
"shear_bbxs" : [bboxes[i]],
}} for i, rank in enumerate(imids)],
"images": [{"prefix": req.dataset_name,
"path": os.path.join(config.images_base_path, cid2filename(engine.id2cid[rank])),
"overlays": {
"rank": req.offset + i + 1,
"name": engine.id2cid[rank],
"loc_features": visualize_local_features(rank, engine, mode='full_geom', n=100, bbox=bboxes[i], query=imid,
user_bbox=user_bbox),
"keypoints": visualize_local_features(rank, engine, mode='point', bbox=bboxes[i]),
"shear_bbxs": [bboxes_path[i]],
}} for i, rank in enumerate(imids)],
}
if mode == "image":
......@@ -172,11 +176,12 @@ def images():
rectangles_over.append(req.query.search_mode.tool_data)
out_data["query_text"] = f"{len(imids)} results in {time.time() - time0:.3f}s for bbox: {str(rectangles_over)}"
out_data["query_image"] = {
"overlays": {
"rectangles_over": rectangles_over,
"text_over" : "search_mode=" + req.query.search_mode.id,
"name" : parse_name(req.query.value.path),
"text_over": "search_mode=" + req.query.search_mode.id,
"name": parse_name(req.query.value.path),
}
}
......@@ -189,7 +194,7 @@ def handle_exception(e):
"""Return JSON instead of HTML for HTTP errors."""
response = e.get_response()
response.data = json.dumps({
"error" : e.name,
"error": e.name,
"message": e.description,
})
response.content_type = "application/json"
......
"""
Engine built on top of precomputed local descriptors.
"""
from __future__ import annotations
import time
import os.path
import json
import traceback
from dataclasses import dataclass
import h5py
import numpy as np
from flask import Flask, request, jsonify
from werkzeug.exceptions import HTTPException
from utilities.path2id import path2id
from data_loading.read_invfile import *
from image_search import unconstrained_search, zoom_in, zoom_out
# Config
DATASETS = {
# Dataset
"bow_600k": {
# Engine
# - when in the singularity container, the path 'horn:/local/vrg3_demo/vrg3_demo/data' is mapped to '/vrg3/data'
"zoom_in": "/local/vrg3_demo/vrg3_demo/data/bow_600k/files/invfile.dat"
},
}
# Input data loading: inverted file
ENGINE = load_invfiles(DATASETS)
# API endpoint
app = Flask(__name__)
def _parse_request(request):
if request.content_type == "application/json":
data = request.get_json()
else:
data = {'data': json.loads(request.form['data'])} if 'data' in request.form else {}
data = data['data']
assert data['dataset_name'] in ENGINE, "Unsupported dataset"
assert data['engine_name'] in ENGINE[data['dataset_name']], "Unsupported network"
return data
@app.route("/capabilities", methods=['POST'])
def capabilities():
data = {"data": {
"search_types": ["image"],
"engine_options": [
{"id": "mode", "label": "Search mode",
"type": "option", "values": ["Unconstrained", "Zoom-in", "Zoom-out"],
"default": "Unconstrained"},
],
"images_overlays": [
{"id": "rank", "label": "Rank", "type": "text", "default": True},
{"id": "name", "label": "Name", "type": "text", "default": True},
{"id": "bbxs", "label": "Bounding box", "type": "rectangles", "default": True},
],
}}
return jsonify(data), 200
@app.route("/images", methods=['POST'])
def images():
time0 = time.time()
data = _parse_request(request)
dataset = ENGINE[data["dataset_name"]]
mode = None
if not data.get("query", None):
# Browsing functionality
# image paths from the chosen dataset
images = next(iter(dataset.values())).paths
np.random.seed(seed=0)
# Show images in a random order
ranks = np.random.permutation(np.arange(len(images)))
mode = "browsing"
bbxs = [[] for i in range(len(ranks))]
elif data["query"]["type"] == "image":
# Search-by-image functionality
assert data["query"]["value"]["prefix"] == data["dataset_name"]
engine = dataset[data["engine_name"]]
images = engine.paths
query_id = path2id(data["query"]["value"]["path"], engine.cid2id)
#raise ValueError(query_id)
if data["engine_options"]["mode"] == "Unconstrained":
# Do unconstrained search
ranks = unconstrained_search(query_id, engine)
bbxs = [[] for i in range(len(ranks))]
elif data["engine_options"]["mode"] == "Zoom-in":
# Do zoom-in search
ranks, bbxs = zoom_in(query_id, engine)
elif data["engine_options"]["mode"] == "Zoom-out":
# Do zoom-out search
ranks, bbxs = zoom_out(query_id, engine)
else:
# Unknown search mode
raise ValueError("Unknown search mode")
# Remove query from results
ranks = ranks[1:]
bbxs = bbxs[1:]
mode = "image"
else:
raise ValueError("Unknown query type")
# Output building
slice = [images[i] for i in ranks[data['offset']:data['offset']+data['limit']]] # Paging
parse_name = lambda x: os.path.splitext(os.path.basename(x))[0]
outdata = {
"results": len(images),
"images": [{"prefix": data['dataset_name'],
"path": x,
"overlays": {
"rank": data['offset']+i+1,
"name": parse_name(x),
"bbxs": bbxs[i],
}} for i, x in enumerate(slice)],
}
if mode == "image":
outdata["query_text"] = "%s results in %.3fs (engine_options: %s)" % (len(images), time.time() - time0, data["engine_options"])
outdata["query_image"] = {"overlays": {"name": parse_name(data["query"]["value"]["path"])}}
return jsonify({"data": outdata}), 200
# Error handling
@app.errorhandler(HTTPException)
def handle_exception(e):
"""Return JSON instead of HTML for HTTP errors."""
response = e.get_response()
response.data = json.dumps({
"error": e.name,
"message": e.description,
})
response.content_type = "application/json"
return response
@app.errorhandler(Exception)
def handle_exception(e):
# pass through HTTP errors
if isinstance(e, HTTPException):
return e
# now you're handling non-HTTP exceptions only
traceback.print_exc()
return jsonify({"error": repr(e)}), 500
"""
Engine built on top of precomputed local descriptors.
"""
from __future__ import annotations
import time
import os.path
import json
import traceback
from dataclasses import dataclass
import h5py
import numpy as np
from flask import Flask, request, jsonify
from werkzeug.exceptions import HTTPException
from utilities.path2id import path2id
from data_loading.read_invfile import *
from image_search import unconstrained_search, zoom_in, zoom_out
# Config
DATASETS = {
# Dataset
"bow_600k": {
# Engine
# - when in the singularity container, the path 'horn:/local/vrg3_demo/vrg3_demo/data' is mapped to '/vrg3/data'
"zoom_in": "/local/vrg3_demo/vrg3_demo/data/bow_600k/files/invfile.dat"
},
}
# Input data loading: inverted file
ENGINE = load_invfiles(DATASETS)
# API endpoint
app = Flask(__name__)
def _parse_request(request):
if request.content_type == "application/json":
data = request.get_json()
else:
data = {'data': json.loads(request.form['data'])} if 'data' in request.form else {}
data = data['data']
assert data['dataset_name'] in ENGINE, "Unsupported dataset"
assert data['engine_name'] in ENGINE[data['dataset_name']], "Unsupported network"
return data
@app.route("/capabilities", methods=['POST'])
def capabilities():
data = {"data": {
"search_types": ["image"],
"engine_options": [
{"id": "mode", "label": "Search mode",
"type": "option", "values": ["Unconstrained", "Zoom-in", "Zoom-out"],
"default": "Unconstrained"},
],
"images_overlays": [
{"id": "rank", "label": "Rank", "type": "text", "default": True},
{"id": "name", "label": "Name", "type": "text", "default": True},
{"id": "bbxs", "label": "Bounding box", "type": "rectangles", "default": True},
],
}}
return jsonify(data), 200
@app.route("/images", methods=['POST'])
def images():
time0 = time.time()
data = _parse_request(request)
dataset = ENGINE[data["dataset_name"]]
mode = None
if not data.get("query", None):
# Browsing functionality
# image paths from the chosen dataset
images = next(iter(dataset.values())).paths
np.random.seed(seed=0)
# Show images in a random order
ranks = np.random.permutation(np.arange(len(images)))
mode = "browsing"
bbxs = [[] for i in range(len(ranks))]
elif data["query"]["type"] == "image":
# Search-by-image functionality
assert data["query"]["value"]["prefix"] == data["dataset_name"]
engine = dataset[data["engine_name"]]
images = engine.paths
query_id = path2id(data["query"]["value"]["path"], engine.cid2id)
#raise ValueError(query_id)
if data["engine_options"]["mode"] == "Unconstrained":
# Do unconstrained search
ranks = unconstrained_search(query_id, engine)
bbxs = [[] for i in range(len(ranks))]
elif data["engine_options"]["mode"] == "Zoom-in":
# Do zoom-in search
ranks, bbxs = zoom_in(query_id, engine)
elif data["engine_options"]["mode"] == "Zoom-out":
# Do zoom-out search
ranks, bbxs = zoom_out(query_id, engine)
else:
# Unknown search mode
raise ValueError("Unknown search mode")
# Remove query from results
ranks = ranks[1:]
bbxs = bbxs[1:]
mode = "image"
else:
raise ValueError("Unknown query type")
# Output building
slice = [images[i] for i in ranks[data['offset']:data['offset']+data['limit']]] # Paging
parse_name = lambda x: os.path.splitext(os.path.basename(x))[0]
outdata = {
"results": len(images),
"images": [{"prefix": data['dataset_name'],
"path": x,
"overlays": {
"rank": data['offset']+i+1,
"name": parse_name(x),
"bbxs": bbxs[i],
}} for i, x in enumerate(slice)],
}
if mode == "image":
outdata["query_text"] = "%s results in %.3fs (engine_options: %s)" % (len(images), time.time() - time0, data["engine_options"])
outdata["query_image"] = {"overlays": {"name": parse_name(data["query"]["value"]["path"])}}
return jsonify({"data": outdata}), 200
# Error handling
@app.errorhandler(HTTPException)
def handle_exception(e):
"""Return JSON instead of HTML for HTTP errors."""
response = e.get_response()
response.data = json.dumps({
"error": e.name,
"message": e.description,
})
response.content_type = "application/json"
return response
@app.errorhandler(Exception)
def handle_exception(e):
# pass through HTTP errors
if isinstance(e, HTTPException):
return e
# now you're handling non-HTTP exceptions only
traceback.print_exc()
return jsonify({"error": repr(e)}), 500
......@@ -12,7 +12,7 @@ datasets:
max_tc: 600
max_MxN: 10
max_spatial: 50
inlier_threshold: 20
inlier_threshold: 5
minimum_score: 0.0
use_query_expansion: True
max_qe_keypoints: 1500
......
......@@ -4,7 +4,7 @@ import h5py
import numpy as np
from loading import Data
from utilities import Rect, cid2filename
from utilities import BBox, cid2filename
from .inverted_chunk import InvertedChunk
......@@ -85,7 +85,7 @@ class Engine:
return InvertedChunk(self.invfile_bytes, offset, wf, self.code_bits, max_bit).decode()
def get_geometries(self, imid: int, bbox: Rect = None):
def get_geometries(self, imid: int, bbox: BBox = None):
"""
Returns the geometries of the given document.
:param bbox: If provided, only the geometries are inside this bounding box are returned.
......@@ -100,10 +100,16 @@ class Engine:
# inverting of the transformation matrix of the ellipses from the initial de-normalizing to the normalizing one
positions[:, 2] = 1. / positions[:, 2]
positions[:, 4] = 1. / positions[:, 4]
positions[:, 3] = -positions[:, 3] * positions[:, 2] * positions[:, 4]
positions[:, 3] = - positions[:, 3] * positions[:, 2] * positions[:, 4]
if bbox is not None:
labels, position = bbox.get_inner(positions[:, 0], positions[:, 1], labels, positions)
x = positions[:, 0]
y = positions[:, 1]
mask = (x > bbox.x1) & (y > bbox.y1) & (x < bbox.x2) & (y < bbox.y2)
labels = labels[mask]
positions = positions[mask]
tfidf, unique_labels, unique_indices, unique_counts = self.get_idf(labels)
......
......@@ -3,7 +3,7 @@ from collections import defaultdict
from engine import Engine
from image_search import search_methods
from loading import load_data
from utilities import Rect, dotdict, read_yaml
from utilities import BBox, dotdict, read_yaml
engines = defaultdict(dict)
datasets = read_yaml("/local/vrg3_demo/vrg3_demo/app/engine_vdvm22/config.yml").datasets
......@@ -30,13 +30,12 @@ engine, config = engines.bow_600k.zoom_in
rectangles_over = []
# Search-by-image functionality
cid = 'b47c1d752619d0b9b4284293b1cb6c41'
imid = engine.cid2id[cid]
imid = 0
width, height = engine.image_sizes[imid]
# Set the bounding box to the full size of the image
bbox = Rect(x2=width, y2=height)
bbox = BBox(x2=width, y2=height)
search_mode = "Unconstrained"
......
from .search_methods import search_methods
#from .image_search import *
#from .spatial_verification import *
#from .utils import *
from .search_methods import *
......@@ -4,56 +4,36 @@ from typing import List, Tuple
import numpy as np
from engine import Engine
from utilities import Rect
from .spatial_verification import query_spatial_verification
from utilities import BBox
from .utils import make_query
def search_methods(imid: int, engine: Engine, bbox: Rect = None, mode="Unconstrained") -> Tuple[List[int], List[np.ndarray]]:
def search_methods(imid: int, engine: Engine, bbox: BBox = None, mode="Unconstrained") -> Tuple[np.ndarray, List[BBox]]:
if bbox is None:
bbox = Rect()
bbox = BBox()
_, idxs, As = query_spatial_verification(imid, bbox, engine)
bboxes = []
for img_id, A in zip(idxs, As):
corner_pts = np.hstack((bbox.corner_points, np.ones((4, 1)))).astype(int)
corner_pts = corner_pts @ A.T
width, height = engine.image_sizes[img_id]
corner_pts[:, 0] /= width
corner_pts[:, 1] /= height
bboxes.append(corner_pts[:, :2].tolist())
if mode == "Unconstrained":
return idxs.tolist(), bboxes
ranks, bboxes = make_query(imid, bbox, engine)
if mode == "Zoom-in":
# The area is given by the determinant (we don't care about the original bbox area as it's multiplied with each determinant)
scores = [get_bbox_area(bboxes[i]) * engine.image_area[imid_] for i, imid_ in enumerate(idxs)]
scores = [bb.area * engine.image_area[r] for r, bb in zip(ranks, bboxes)]
sort = np.argsort(scores)[::-1]
idxs = [idxs[i] for i in sort]
ranks = [ranks[i] for i in sort]
bboxes = [bboxes[i] for i in sort]
elif mode == "Zoom-out":
# The area is given by the determinant (we don't care about the original bbox area as it's multiplied with each determinant)
scores = [get_bbox_area(bboxes[i]) / engine.image_area[imid_] for i, imid_ in enumerate(idxs)]
scores = [bb.area / engine.image_area[r] for r, bb in zip(ranks, bboxes)]
sort = np.argsort(scores)
idxs = [idxs[i] for i in sort]
ranks = [ranks[i] for i in sort]
bboxes = [bboxes[i] for i in sort]
elif mode == "Unconstrained":
pass
else:
raise ValueError("Unknown search mode")
return idxs, bboxes
def get_bbox_area(points) -> float: