This highlights the need to and the method for reducing workload for populating the drone world catalog based on aerial drone imagery.
#! /usr/bin/python
import json
from azure.search.documents import SearchClient
from azure.core.credentials import AzureKeyCredential
from azure.ai.vision.imageanalysis import ImageAnalysisClient
from azure.search.documents.models import (
VectorizedQuery,
VectorizableTextQuery
)
from dedup import ImageDeduplicator
from tenacity import retry, stop_after_attempt, wait_fixed
import os
import re
import sys
import time
search_endpoint = os.environ["AZURE_SEARCH_SERVICE_ENDPOINT"]
api_version = os.getenv("AZURE_SEARCH_API_VERSION")
search_api_key = os.getenv("AZURE_SEARCH_ADMIN_KEY")
index_name = os.getenv("AZURE_SEARCH_INDEX_NAME", "index00")
credential = AzureKeyCredential(search_api_key)
dest_index_name = os.getenv("AZURE_SEARCH_02_INDEX_NAME", "index02")
vision_api_key = os.getenv("AZURE_AI_VISION_API_KEY")
vision_api_version = os.getenv("AZURE_AI_VISION_API_VERSION")
vision_region = os.getenv("AZURE_AI_VISION_REGION")
vision_endpoint = os.getenv("AZURE_AI_VISION_ENDPOINT")
source_url_template = os.getenv("AZURE_SOURCE_SAS_URI")
destination_url_template = os.getenv("AZURE_DESTINATION_SAS_URI")
sys.path.insert(0, os.path.abspath(".."))
from visionprocessor.vectorizer import vectorize_image, analyze_image
deduplicator = ImageDeduplicator()
# Initialize SearchClient
search_client = SearchClient(
endpoint=search_endpoint,
index_name=index_name,
credential=AzureKeyCredential(search_api_key)
)
destination_client = SearchClient(
endpoint=search_endpoint,
index_name=dest_index_name,
credential=AzureKeyCredential(search_api_key)
)
vision_credential = AzureKeyCredential(vision_api_key)
analysis_client = ImageAnalysisClient(vision_endpoint, vision_credential)
import cv2
import numpy as np
import requests
from io import BytesIO
from azure.storage.blob import BlobClient
def read_image_from_blob(sas_url):
"""Reads an image from Azure Blob Storage using its SAS URL."""
response = None
try:
response = requests.get(sas_url)
except Exception as e:
print(f"Error from requests.get: {e}")
if response.status_code == 200:
image_array = np.asarray(bytearray(response.content), dtype=np.uint8)
image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
return image
else:
# raise Exception(f"Failed to fetch image. Status code: {response.status_code}")
return None
def upload_image_to_blob(clipped_image, sas_url):
"""Uploads the clipped image to Azure Blob Storage using its SAS URL."""
_, encoded_image = cv2.imencode(".jpg", clipped_image)
blob_client = BlobClient.from_blob_url(sas_url)
blob_client.upload_blob(encoded_image.tobytes(), overwrite=True)
# print("Clipped image uploaded successfully.")
def save_or_display(clipped_image, destination_file):
cv2.imwrite(destination_file, clipped_image)
cv2.imshow("Clipped Image", clipped_image)
cv2.waitKey(0)
cv2.destroyAllWindows()
def clip_image(image, bounding_box):
# Extract bounding box parameters
x, y, width, height = bounding_box
# Clip the region using slicing
clipped_image = image[y:y+height, x:x+width]
return clipped_image
def prepare_json_string_for_load(text):
text = text.replace("\"", "'")
text = text.replace("{'", "{\"")
text = text.replace("'}", "\"}")
text = text.replace(" '", " \"")
text = text.replace("' ", "\" ")
text = text.replace(":'", ":\"")
text = text.replace("':", "\":")
text = text.replace(",'", ",\"")
text = text.replace("',", "\",")
return re.sub(r'\n\s*', '', text)
def to_string(bounding_box):
return f"{bounding_box['x']},{bounding_box['y']},{bounding_box['w']},{bounding_box['h']}"
def is_duplicate_image(deduplicator, image):
value = deduplicator.is_duplicate(image)
return value
def is_visited(deduplicator, vector):
value = deduplicator.is_visited(vector)
return value
def is_existing(deduplicator, vector):
start_time = time.time()
value = deduplicator.is_existing(destination_client, vector)
end_time = time.time()
elapsed_time = end_time - start_time
print(f"Elapsed time for is_existing: {elapsed_time:.3f} seconds")
return value
@retry(stop=stop_after_attempt(5), wait=wait_fixed(60))
def upload(document):
try:
upload_results = destination_client.upload_documents([document])
error = ','.join([upload_result.error_message for upload_result in upload_results if upload_result.error_message]).strip(",")
if error:
print(error)
except HttpResponseError as e:
print(f"Error from upload: {e}")
raise
# Example usage
def shred(entry_id):
source_file=entry_id
source_sas_url = source_url_template.replace("{source_file}", source_file)
print(entry_id)
entry = search_client.get_document(key=entry_id) # , select=["id", "description"])
id=entry['id']
description_text=entry['description']
tags = entry['tags']
title = entry['title']
description_json = None
try:
description_text = prepare_json_string_for_load(entry["description"]).replace('""','')
description_json = json.loads(description_text)
except Exception as e:
print(description_text)
print(f"{entry_id}: parsing error: {e}")
if description_json == None:
print("Description could not be parsed.")
return
if description_json and description_json["_data"] and description_json["_data"]["denseCaptionsResult"] and description_json["_data"]["denseCaptionsResult"]["values"]:
objectid = 0
for item in description_json["_data"]["denseCaptionsResult"]["values"]:
objectid += 1
if objectid == 1:
continue
destination_file=source_file+f"-{objectid:04d}"
destination_sas_url = destination_url_template.replace("{destination_file}", destination_file)
box = item.get("boundingBox", None)
print(f"{destination_file}: {box}")
if box:
bounding_box = (box["x"], box["y"], box["w"], box["h"])
# Read image from Azure Blob
image = read_image_from_blob(source_sas_url)
if image.any() == False:
print(f"{destination_file} not found.")
continue
# Clip image
clipped = clip_image(image, bounding_box)
# Upload clipped image to Azure Blob
upload_image_to_blob(clipped, destination_sas_url)
vector = vectorize_image(destination_sas_url, vision_api_key, "eastus")
vector = np.pad(vector, (0, 1536 - len(vector)), mode='constant')
print("checking existing")
if vector.any() and is_existing(deduplicator, vector) == False:
print(f"Match does not exist for {destination_file}.")
else:
print(f"Match exists for {destination_file}")
else:
print("no objects detected")
for number in range(5412, 5413):
entry_id = f"{number:06d}"
shred(entry_id)
With deduplicator.is_existing() method as:
import cv2
import imagehash
import numpy as np
from PIL import Image
from collections import deque
from azure.search.documents.models import (
VectorizedQuery,
VectorizableTextQuery
)
class ImageDeduplicator:
def __init__(self, buffer_size=100):
"""Initialize a ring buffer for tracking image hashes."""
self.buffer_size = buffer_size
self.hash_buffer = deque(maxlen=buffer_size)
self.vector_buffer = deque(maxlen=buffer_size)
def compute_hash(self, image):
"""Compute perceptual hash of an image."""
return imagehash.phash(Image.fromarray(image))
def is_existing(self, external_vector_client, vector):
vector_query = VectorizedQuery(vector=vector,
k_nearest_neighbors=3,
exhaustive=False,
fields = "vector")
results = external_vector_client.search(
search_text=None,
vector_queries= [vector_query],
select=["id", "description","vector"],
# select='id,description,vector',
include_total_count=True,
top=4
)
if results != None and results.get_count() > 0:
best = 0
id = None
for match in results:
# print(f"{match['id']} found." + ",".join([key for key in match.keys()]))
match_vector = match["vector"]
score = self.cosine_similarity(vector, match_vector)
# print(f"score={score}")
if score > best:
id = match['id']
best = score
else:
continue
matches = ','.join([match['id'] for match in results]).strip(',')
print(f"matches: {matches}")
if best > 0.8:
print(f"match found with score {best} for {id}.")
return True
else:
print("no match found.")
return False
def get_hash_buffer_len(self):
return len(self.hash_buffer)
def get_vector_buffer_len(self):
return len(self.vector_buffer)
def cosine_similarity(self, vec1, vec2):
"""Computes cosine similarity between two vectors."""
dot_product = np.dot(vec1, vec2)
norm_vec1 = np.linalg.norm(vec1)
norm_vec2 = np.linalg.norm(vec2)
return dot_product / (norm_vec1 * norm_vec2)
And results as follows:
005412
005412-0002: {'x': 986, 'y': 49, 'w': 563, 'h': 526}
checking existing
000370-0002
001225-0002
002703-0002
match found with score 0.9856458102556909 for 000370-0002.
Elapsed time for is_existing: 0.607 seconds
Match exists for 005412-0002
005412-0003: {'x': 1363, 'y': 400, 'w': 422, 'h': 373}
checking existing
001784-0006
004981-0004
014676-0003
match found with score 0.9866765401858795 for 001784-0006.
Elapsed time for is_existing: 0.291 seconds
Match exists for 005412-0003
005412-0004: {'x': 0, 'y': 0, 'w': 1896, 'h': 1050}
checking existing
005412-0004
003169-0006
012227-0006
match found with score 0.9999997660907427 for 005412-0004.
Elapsed time for is_existing: 0.239 seconds
Match exists for 005412-0004
005412-0005: {'x': 1110, 'y': 705, 'w': 403, 'h': 363}
checking existing
005412-0005
004463-0007
004980-0008
match found with score 1.0000000000000002 for 005412-0005.
Elapsed time for is_existing: 0.310 seconds
Match exists for 005412-0005
005412-0006: {'x': 1279, 'y': 213, 'w': 77, 'h': 76}
checking existing
005412-0006
014698-0009
013267-0008
match found with score 1.0000000000000002 for 005412-0006.
Elapsed time for is_existing: 0.288 seconds
Match exists for 005412-0006
005412-0007: {'x': 266, 'y': 717, 'w': 69, 'h': 59}
checking existing
005412-0007
012227-0004
015072-0007
match found with score 1.0 for 005412-0007.
Elapsed time for is_existing: 0.314 seconds
Match exists for 005412-0007
005412-0008: {'x': 612, 'y': 441, 'w': 160, 'h': 184}
checking existing
005412-0008
004989-0009
001226-0003
match found with score 1.0 for 005412-0008.
Elapsed time for is_existing: 0.289 seconds
Match exists for 005412-0008
005412-0009: {'x': 775, 'y': 381, 'w': 68, 'h': 66}
checking existing
005412-0009
013213-0005
005416-0004
match found with score 0.9999997252673284 for 005412-0009.
Elapsed time for is_existing: 0.319 seconds
Match exists for 005412-0009
005412-0010: {'x': 4, 'y': 330, 'w': 76, 'h': 66}
checking existing
005412-0010
004464-0007
015072-0005
match found with score 1.0 for 005412-0010.
Elapsed time for is_existing: 0.269 seconds
Match exists for 005412-0010
At nearly 0.3 seconds per object existence check in the drone world catalog and about ten objects per image in a set of 17533 images in a single tour of a drone, this comes to 17533 * 10 * 0.3 / (60*60) hours = 14.61 hours. So workload reduction is called for and images that have even 20% matches or more with existing objects in the catalog can be discarded unless a thresholded time-span is exceeded.
And this even works for comparisons as shown:
To generate a preview video, we could use something like:
def get_preview_url(video_id, access_token):
insights_url = f"https://api.videoindexer.ai/{LOCATION}/Accounts/{ACCOUNT_ID}/Videos/{video_id}/Index?accessToken={access_token}"
response = requests.get(insights_url)
insights = response.json()
preview_url = insights.get('summarizedInsights', {}).get('previewUrl')
return preview_url
