Sending & querying images¶
Sending images¶
In this example we will send images with labels to SEEREP.
In order to save images, we need to mandatorily provide the intrinsics of the camera used to capture them. After the successfully saving the camera intrinsics, we need to provide the uuid of it along with the images. SEEREP will ensure that the Camera Intrinsics UUID provided with an image has a UUID stored against it.
Additionally we add some coordinate transformations at the end.
Source: examples/python/gRPC/images/gRPC_pb_sendLabeledImage.py
#!/usr/bin/env python3
# NOTE: This file is referenced in the following mkdocs files:
# images.md
# Any changes done in here will be reflected in there
import time
import uuid
from typing import List, Optional, Tuple
import numpy as np
from google.protobuf import empty_pb2
from grpc import Channel
from seerep.pb import camera_intrinsics_pb2 as cameraintrinsics
from seerep.pb import (
camera_intrinsics_service_pb2_grpc as camintrinsics_service,
)
from seerep.pb import image_pb2 as image
from seerep.pb import image_service_pb2_grpc as imageService
from seerep.pb import label_category_pb2, label_pb2
from seerep.pb import meta_operations_pb2_grpc as metaOperations
from seerep.pb import projectCreation_pb2 as projectCreation
from seerep.pb import tf_service_pb2_grpc as tfService
from seerep.pb import transform_stamped_pb2 as tf
from seerep.util.common import get_gRPC_channel
def send_labeled_images(
target_proj_uuid: Optional[str] = None,
grpc_channel: Channel = get_gRPC_channel(),
) -> Tuple[
List[List[image.Image]], List[int], cameraintrinsics.CameraIntrinsics
]:
"""sends test images via the given grpc_channel to the specified target
project uuid"""
# 1. Get gRPC service objects
stub = imageService.ImageServiceStub(grpc_channel)
stubTf = tfService.TfServiceStub(grpc_channel)
stubMeta = metaOperations.MetaOperationsStub(grpc_channel)
stubCI = camintrinsics_service.CameraIntrinsicsServiceStub(grpc_channel)
# 2. Check if we have an existing test project (or target_proj_uuid is set),
# if not, one is created.
if target_proj_uuid is None:
# 3. Get all projects from the server
response = stubMeta.GetProjects(empty_pb2.Empty())
for project in response.projects:
print(project.name + " " + project.uuid)
if project.name == "testproject":
target_proj_uuid = project.uuid
if target_proj_uuid is None:
# 4. create a project
creation = projectCreation.ProjectCreation(
name="testproject", mapFrameId="map"
)
projectCreated = stubMeta.CreateProject(creation)
target_proj_uuid = projectCreated.uuid
theTime = int(time.time())
#####
# A valid camera intrinsics UUID is needed here for succesful storage
# of Images
# Add new Camera Intrinsics with placeholder data
ciuuid = str(uuid.uuid4())
camin = cameraintrinsics.CameraIntrinsics()
camin.header.stamp.seconds = 4
camin.header.stamp.nanos = 3
camin.header.frame_id = "camintrinsics"
camin.header.uuid_project = target_proj_uuid
camin.header.uuid_msgs = ciuuid
camin.region_of_interest.x_offset = 2
camin.region_of_interest.y_offset = 1
camin.region_of_interest.height = 5
camin.region_of_interest.width = 4
camin.region_of_interest.do_rectify = 4
camin.height = 5
camin.width = 4
camin.distortion_model = "plumb_bob"
camin.distortion.extend(list(range(0, 3)))
camin.intrinsic_matrix.extend([3, 4, 5, 10, 7, 8, 9, 10, 11])
camin.rectification_matrix.extend([3, 4, 5, 6, 7, 8, 9, 10, 11])
camin.projection_matrix.extend([3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
camin.binning_x = 6
camin.binning_y = 7
camin.maximum_viewing_distance = 5
stubCI.TransferCameraIntrinsics(camin)
# 5. Create ten images
# for debugging and testing this example add all sent images to a list
sent_images_list: List[Tuple[str, image.Image]] = []
for n in range(10):
theImage = image.Image()
rgb = []
lim = 256 # 256 x 256 pixels
for i in range(lim):
for j in range(lim):
x = float(i) / lim
y = float(j) / lim
z = float(j) / lim
r = np.ubyte((x * 255.0 + n) % 255)
g = np.ubyte((y * 255.0 + n) % 255)
b = np.ubyte((z * 255.0 + n) % 255)
rgb.append(r)
rgb.append(g)
rgb.append(b)
# Add image meta-data
theImage.header.frame_id = "camera"
theImage.header.stamp.seconds = theTime + n
theImage.header.stamp.nanos = 0
theImage.header.uuid_project = target_proj_uuid
theImage.height = lim
theImage.width = lim
theImage.encoding = "rgb8"
theImage.step = 3 * lim
theImage.uuid_camera_intrinsics = ciuuid
# Add image data
theImage.data = bytes(rgb)
# 5. create label
labelStr = ["label1", "label2"]
for iCategory in ["category A", "category B"]:
labelsCategory = label_category_pb2.LabelCategory()
labelsCategory.category = iCategory
labelsCategory.datumaroJson = "a very valid datumaro json"
for labelAct in labelStr:
label = label_pb2.Label()
label.label = labelAct
label.labelIdDatumaro = 1
label.instanceUuid = str(uuid.uuid4())
label.instanceIdDatumaro = 22
labelsCategory.labels.append(label)
theImage.labels.append(labelsCategory)
# 8. Send image to the server
uuidImg = stub.TransferImage(theImage)
# also add them to the list
sent_images_list.append((uuidImg.message, theImage))
# a list for debugging and testing, if interpolated coordinates
# according to timestamp of the images and tfs are correct
tf_times: List[int] = []
# 8. Add coordinate transformations and send them to the server
theTf = tf.TransformStamped()
theTf.header.frame_id = "map"
theTf.header.stamp.seconds = theTime
theTf.header.uuid_project = target_proj_uuid
theTf.child_frame_id = "camera"
theTf.transform.translation.x = 1
theTf.transform.translation.y = 2
theTf.transform.translation.z = 3
theTf.transform.rotation.x = 0
theTf.transform.rotation.y = 0
theTf.transform.rotation.z = 0
theTf.transform.rotation.w = 1
stubTf.TransferTransformStamped(theTf)
tf_times.append(theTf.header.stamp.seconds)
theTf.header.stamp.seconds = theTime + 10
theTf.transform.translation.x = 100
theTf.transform.translation.y = 200
theTf.transform.translation.z = 300
stubTf.TransferTransformStamped(theTf)
tf_times.append(theTf.header.stamp.seconds)
# return sent data
return sent_images_list, tf_times, camin
if __name__ == "__main__":
sent_image_ls_data, _, _ = send_labeled_images()
camera_intrinsics_allimgs = {
intrins_uuid[1].uuid_camera_intrinsics
for intrins_uuid in sent_image_ls_data
}
# print statement to seperate the messages of the function
print()
print(
f"camera intrinsics will be saved against the uuid(s): "
f"{camera_intrinsics_allimgs}"
)
img_uuids = [img[0] for img in sent_image_ls_data]
for i, img_uuid in enumerate(img_uuids):
print(f"the uuid of the sent image number {i} is: {img_uuid}")
Output:
testproject 842de425-2d50-4adf-8aa3-6df257a7c76c
testproject ff739be8-cf0c-4657-bff0-f66f3e7f578d
camera intrinsics will be saved against the uuid(s): {'909c8439-36b6-44ff-9561-f7921de5d8e8'}
the uuid of the sent image number 0 is: 261a7b2c-297b-42db-93da-770e5f7f53cf
the uuid of the sent image number 1 is: feddb62d-fdfa-494a-8e4a-e1b2209134a6
the uuid of the sent image number 2 is: 6956ea01-b834-461f-85cf-5621e04f02fb
the uuid of the sent image number 3 is: 111e62dd-9aba-4fde-ae3d-ed3e3b6041f8
the uuid of the sent image number 4 is: 67138174-08a9-4943-848a-736bddd755b9
the uuid of the sent image number 5 is: 5e139007-9eea-4e71-9c5a-44f08c93027e
the uuid of the sent image number 6 is: c10eff49-4412-425d-99ef-d8b3f95c3806
the uuid of the sent image number 7 is: 708be3d8-7ddb-4ef5-9b00-37021b7b83ff
the uuid of the sent image number 8 is: 4ce47c5c-60d2-4f78-810f-ac730136f462
the uuid of the sent image number 9 is: 6c9ca46b-84fb-40d2-8523-2ad07284a43c
Query images¶
Now we will query the previously send images with some criteria. Possible query parameters are:
2D Polygon (spatial query)¶
Spatial queries in SEEREP are performed using a 2D polygon. This polygon should be simple (no more than 2 vertices on the same edge) and convex (no edges curving inward). This 2D polygon lies on a interval on the z-axis defined through a z-coordinate point and a height value. Queries are performed by forming an encompassing axis aligned bounding box from the polygon. This can lead to an AABB larger than the polygon and poses the potential problem of returning results to the user which are not fully inside the query polygon. That problem is resolved by providing a boolean variable called fullyEncapsulated`. If false, resultant polygons, which are partially inside the query polygon are also returned.
A time interval (temporal query)¶
Temporal queries in SEEREP are performed using a time interval. When using image
queries the stamp in the header of those images is used and when that time
lies in the interval the image is returned as part of the response. The interval
is closed.
Labels (semantic query)¶
ProjectUuids¶
Only performs the query in these included projects specified by their uuids.
WithoutData¶
If the pixel data of the image should not be returned in order to save bandwith.
InMapFrame¶
Whether the query is done in the map frame. If not, the provided polygon for the spatial query will be transformed from the geodesic coordinates of the project into the map frame beforehand.
Example code for querying images¶
Source: examples/python/gRPC/images/gRPC_fb_queryImage.py
#!/usr/bin/env python3
# NOTE: This file is referenced in the following mkdocs files:
# images.md
# Any changes done in here will be reflected in there
import sys
from typing import List
import flatbuffers
from grpc import Channel
from seerep.fb import Image
from seerep.fb import image_service_grpc_fb as imageService
from seerep.util.common import get_gRPC_channel
from seerep.util.fb_helper import (
create_label,
create_label_category,
createPoint2d,
createPolygon2D,
createQuery,
createTimeInterval,
createTimeStamp,
getOrCreateProject,
)
def query_images_raw(
fbb: flatbuffers.Builder,
grpc_channel: Channel,
target_proj_uuid: str,
*args,
**kwargs,
) -> List[bytearray]:
"""
Query images from SEEREP and return the raw FlatBuffer buffers.
Args:
grpc_channel (Channel): gRPC channel to communicate with the server.
target_proj_uuid (str): UUID of the target project.
*args: Variable args to pass to the query.
**kwargs: Keyword args to pass to the query.
Returns:
List[bytearray]: A list buffers of images matching the query.
"""
image_service_stub = imageService.ImageServiceStub(grpc_channel)
query = createQuery(fbb, *args, projectUuids=[target_proj_uuid], **kwargs)
fbb.Finish(query)
query_buf = fbb.Output()
return image_service_stub.GetImage(bytes(query_buf))
def query_images(
fbb: flatbuffers.Builder,
grpc_channel: Channel,
target_proj_uuid: str,
*args,
**kwargs,
) -> List[Image.Image]:
"""
Query images from SEEREP.
Args:
grpc_channel (Channel): gRPC channel to communicate with the server.
target_proj_uuid (str): The UUID of the target project.
*args: Variable args to pass to the query.
**kwargs: Keyword args to pass to the query.
Returns:
List[Image.Image]: A list of images matching the query.
"""
return [
Image.Image.GetRootAs(img)
for img in query_images_raw(
fbb, grpc_channel, target_proj_uuid, *args, **kwargs
)
]
if __name__ == "__main__":
fbb = flatbuffers.Builder()
grpc_channel = get_gRPC_channel()
project_uuid = getOrCreateProject(fbb, grpc_channel, "testproject")
# create the data for the query
scale = 100
vertices = [
createPoint2d(fbb, x * scale, y * scale)
for x, y in [(-1.0, -1.0), (-1.0, 1.0), (1.0, 1.0), (1.0, -1.0)]
]
polygon_2d = createPolygon2D(fbb, 700, -100, vertices)
time_min = createTimeStamp(fbb, 1610549273, 0)
time_max = createTimeStamp(fbb, 1938549273, 0)
time_interval = createTimeInterval(fbb, time_min, time_max)
project_uuids = [project_uuid]
labels = [
create_label(builder=fbb, label=label_str, label_id=i)
for i, label_str in enumerate(["label1", "label2"])
]
labelsCategory = [
create_label_category(
builder=fbb,
labels=labels,
datumaro_json="a very valid datumaro json",
category="category A",
)
]
matching_images = query_images(
fbb,
grpc_channel,
project_uuid,
# polygon2d=polygon_2d,
polygon2dSensorPos=polygon_2d,
labels=labelsCategory,
withoutData=True,
fullyEncapsulated=False,
inMapFrame=True,
)
if matching_images is None or len(matching_images) == 0:
print("No images matched the query.")
sys.exit()
print(f"Number of images matching the query: {len(matching_images)}")
for img in matching_images:
print(
"------------------------------------------------------------------"
)
print(f"Msg UUID: {img.Header().UuidMsgs().decode('utf-8')}")
print(f"Number of labels: {img.LabelsLength()}")
if img.LabelsLength() > 0:
print(
"First label: "
+ img.Labels(0).Labels(0).Label().decode("utf-8")
)
print("------------------------------------------------------------------")
After sending the images, executing the query script results in the following output:
count of images: 10
------------------------------------------------------------------
uuidmsg: 111e62dd-9aba-4fde-ae3d-ed3e3b6041f8
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 261a7b2c-297b-42db-93da-770e5f7f53cf
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 4ce47c5c-60d2-4f78-810f-ac730136f462
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 5e139007-9eea-4e71-9c5a-44f08c93027e
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 67138174-08a9-4943-848a-736bddd755b9
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 6956ea01-b834-461f-85cf-5621e04f02fb
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 6c9ca46b-84fb-40d2-8523-2ad07284a43c
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: 708be3d8-7ddb-4ef5-9b00-37021b7b83ff
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: c10eff49-4412-425d-99ef-d8b3f95c3806
count of labels: 2
first label: label1
------------------------------------------------------------------
uuidmsg: feddb62d-fdfa-494a-8e4a-e1b2209134a6
count of labels: 2
first label: label1
------------------------------------------------------------------
Source: examples/python/gRPC/images/gRPC_pb_queryImage.py
#!/usr/bin/env python3
# NOTE: This file is referenced in the following mkdocs files:
# images.md
# Any changes done in here will be reflected in there
import sys
from typing import List, Optional
from google.protobuf import empty_pb2
from grpc import Channel
from seerep.pb import image_pb2 as image
from seerep.pb import image_service_pb2_grpc as imageService
from seerep.pb import label_category_pb2, label_pb2
from seerep.pb import meta_operations_pb2_grpc as metaOperations
from seerep.pb import point2d_pb2 as point2d
from seerep.pb import query_pb2 as query
from seerep.util.common import get_gRPC_channel
def query_images(
target_project_uuid: Optional[str] = None,
grpc_channel: Channel = get_gRPC_channel(),
) -> List[image.Image]:
# 1. Get gRPC service objects
stub = imageService.ImageServiceStub(grpc_channel)
stubMeta = metaOperations.MetaOperationsStub(grpc_channel)
# 3. Check if we have an existing test project, if not, we stop here
if target_project_uuid is None:
# 2. Get all projects from the server
response = stubMeta.GetProjects(empty_pb2.Empty())
for project in response.projects:
print(project.name + " " + project.uuid + "\n")
if project.name == "testproject":
target_project_uuid = project.uuid
if target_project_uuid is None:
print("""
No project with name 'testproject' found! Execute
gRPC_pb_sendLabeledImage.py beforehand!
""")
sys.exit()
# 4. Create a query with parameters
theQuery = query.Query()
theQuery.projectuuid.append(target_project_uuid)
theQuery.polygon.z = -200
theQuery.polygon.height = 800
theQuery.inMapFrame = True
scale = 150
vertices = [
point2d.Point2D(x=x, y=y)
for x, y in [
(-scale, -scale),
(-scale, scale),
(scale, scale),
(scale, -scale),
]
]
theQuery.polygon.vertices.extend(vertices)
# since epoche
theQuery.timeinterval.time_min.seconds = 1638549273
theQuery.timeinterval.time_min.nanos = 0
theQuery.timeinterval.time_max.seconds = 1938549273
theQuery.timeinterval.time_max.nanos = 0
# labels
labelsCategory = label_category_pb2.LabelCategory()
labelsCategory.category = "category A"
label = label_pb2.Label()
label.label = "label1"
labelsCategory.labels.append(label)
theQuery.labelCategory.append(labelsCategory)
# theQuery.inMapFrame = True
theQuery.fullyEncapsulated = False
# 5. Query the server for images matching the query and return them
return list(stub.GetImage(theQuery))
if __name__ == "__main__":
queried_imgs = query_images()
print(f"count of images {len(queried_imgs)}")
for img in queried_imgs:
print(
f"uuidmsg: {img.header.uuid_msgs}"
+ "\n"
+ f"first label: {img.labels[0].labels[0].label}"
)
After sending the images, executing the query script results in the following output:
count of images 8
uuidmsg: 111e62dd-9aba-4fde-ae3d-ed3e3b6041f8
first label: label1
uuidmsg: 261a7b2c-297b-42db-93da-770e5f7f53cf
first label: label1
uuidmsg: 5e139007-9eea-4e71-9c5a-44f08c93027e
first label: label1
uuidmsg: 67138174-08a9-4943-848a-736bddd755b9
first label: label1
uuidmsg: 6956ea01-b834-461f-85cf-5621e04f02fb
first label: label1
uuidmsg: 708be3d8-7ddb-4ef5-9b00-37021b7b83ff
first label: label1
uuidmsg: c10eff49-4412-425d-99ef-d8b3f95c3806
first label: label1
uuidmsg: feddb62d-fdfa-494a-8e4a-e1b2209134a6
first label: label1