Aerial drone video footage to knowledge.
For initiatives such as DVSA, governance at scale becomes economically viable only when video can be managed as knowledge rather than simply stored as footage. The following sample explains how:
index_to_graph.py reads the frame documents in an Azure AI Search index (schema: core/azure/index_schema.py) and builds a spatio-temporal knowledge graph with networkx.
Node From index field
Account, Video account_id, video_id (or the <account>/images/<video>/ blob path)
Frame id, frame_number (or frame<N> in path), timestamp, caption
Object objects plus description.objects from Azure AI Vision
Concept tags, labels, description.tags
Place location
Cell geotags, snapped to a lat/lon grid (default ~110 m)
index_to_graph.py:
from __future__ import annotations
import argparse
import json
import math
import os
import re
from collections import Counter, defaultdict
from datetime import datetime, timezone
from itertools import combinations
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
import networkx as nx
# Fields to pull from the index. ``vector`` is deliberately left out: the graph
# needs semantics and coordinates, not 1536 floats per frame.
SELECT_FIELDS = [
"id", "account_id", "video_id", "title", "section", "frame_number",
"prev_id", "next_id", "path", "imageurl", "created", "timestamp",
"caption", "description", "labels", "tags", "objects", "annotations",
"boundingbox", "geotags", "location", "user", "session", "version",
"acl_users", "acl_groups",
]
# 3 decimal places of lat/lon is roughly a 110 m cell: a sensible "same place"
# for low-altitude drone frames. Raise it for tighter neighbourhoods.
DEFAULT_CELL_PRECISION = 3
# A later frame in the same cell counts as a revisit when it belongs to another
# video, or comes this many seconds (or frames, without timestamps) later.
REVISIT_MIN_GAP = 60
_PATH_RE = re.compile(r"(?:^|/)([^/?]+)/images/([^/?]+)/frame(\d+)\.\w+")
_ID_RE = re.compile(r"-(\d+)$")
_NUM_RE = re.compile(r"-?\d+(?:\.\d+)?")
# --------------------------------------------------------------------------
# 1. Fetch documents from Azure AI Search
# --------------------------------------------------------------------------
def fetch_index_documents(search_client, account_id: Optional[str] = None,
video_id: Optional[str] = None,
user: Optional[str] = None) -> List[Dict[str, Any]]:
"""Page through every frame document for an account (and optionally a video).
Uses the same scoping the connected search agent uses in
``core/azure/agents.py``: ``account_id eq '<id>'`` plus a ``startswith``
on the blob path for one video. ``user`` adds an ACL check.
"""
clauses = []
if account_id:
clauses.append(f"account_id eq '{_odata(account_id)}'")
if video_id:
clauses.append(
f"startswith(path, '{_odata(account_id)}/images/{_odata(video_id)}/')"
f" or video_id eq '{_odata(video_id)}'")
if user:
clauses.append(f"acl_users/any(u: u eq '{_odata(user)}')")
flt = " and ".join(f"({c})" for c in clauses) or None
results = search_client.search(search_text="*", filter=flt,
select=SELECT_FIELDS, top=100000)
return [dict(r) for r in results] # the SDK pages transparently
def _odata(value: str) -> str:
return str(value).replace("'", "''")
def search_client_from_env(index_name: str):
"""A ``SearchClient`` from the same env vars ``AzureEnvironmentConfig`` reads."""
from azure.core.credentials import AzureKeyCredential # noqa: PLC0415
from azure.search.documents import SearchClient # noqa: PLC0415
return SearchClient(endpoint=os.environ["AZURE_SEARCH_ENDPOINT"],
index_name=index_name,
credential=AzureKeyCredential(os.environ["AZURE_SEARCH_ADMIN_KEY"]))
# --------------------------------------------------------------------------
# 2. Normalize one index document into a frame record
# --------------------------------------------------------------------------
def normalize(doc: Dict[str, Any]) -> Dict[str, Any]:
"""Flatten a raw index document into the fields the graph needs."""
path = doc.get("path") or doc.get("imageurl") or ""
m = _PATH_RE.search(path.split("?")[0])
path_account, path_video, path_frame = (m.groups() if m else (None, None, None))
# Prefer the blob path's frame<N>: the id is <account>-<N:04d> with no video
# in it, so it is only a fallback.
frame_number = doc.get("frame_number")
if frame_number is None:
idm = _ID_RE.search(doc.get("id", ""))
frame_number = int(path_frame) if path_frame else (int(idm.group(1)) if idm else None)
desc = _parse_json(doc.get("description"))
caption = doc.get("caption") or desc.get("caption") or ""
tags = _clean(list(doc.get("tags") or []) + list(desc.get("tags") or []))
labels = _clean(doc.get("labels") or [])
objects = _clean(_split_objects(doc.get("objects")) + list(desc.get("objects") or []),
keep_dupes=True)
return {
"id": doc["id"],
"account_id": str(doc.get("account_id") or path_account or ""),
"video_id": str(doc.get("video_id") or path_video or "unknown"),
"title": doc.get("title") or "",
"section": doc.get("section") or "",
"frame_number": frame_number,
"prev_id": doc.get("prev_id"),
"next_id": doc.get("next_id"),
"time": _parse_time(doc.get("timestamp")),
"created": doc.get("created"),
"caption": caption,
"dense_captions": list(desc.get("dense_captions") or []),
"tags": tags,
"labels": labels,
"objects": objects, # duplicates kept = instance counts
"annotations": doc.get("annotations") or "",
"boundingbox": doc.get("boundingbox") or "",
"latlon": _parse_latlon(doc.get("geotags")),
"location": (doc.get("location") or "").strip(),
"path": path.split("?")[0], # never store SAS tokens in a graph
"user": doc.get("user") or "",
"acl_users": list(doc.get("acl_users") or []),
}
def _parse_json(text: Any) -> Dict[str, Any]:
if isinstance(text, dict):
return text
try:
value = json.loads(text) if text else {}
return value if isinstance(value, dict) else {}
except (TypeError, ValueError):
return {}
def _split_objects(value: Any) -> List[str]:
if not value:
return []
if isinstance(value, list):
return value
parsed = _parse_json(value) if value.strip().startswith("{") else None
if parsed:
return list(parsed.get("objects") or [])
try:
as_list = json.loads(value)
if isinstance(as_list, list):
return [str(v) for v in as_list]
except ValueError:
pass
return re.split(r"[,;|]", value)
def _clean(items: Iterable[Any], keep_dupes: bool = False) -> List[str]:
out = [str(i).strip().lower() for i in items if str(i).strip()]
return out if keep_dupes else sorted(set(out))
def _parse_time(value: Any) -> Optional[float]:
if not value:
return None
if isinstance(value, datetime):
return value.timestamp()
try:
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
except ValueError:
return None
def _parse_latlon(value: Any) -> Optional[Tuple[float, float]]:
"""``"('47.6', '-122.1')"``, ``"47.6,-122.1"`` or ``"('', '')"`` -> (lat, lon)."""
nums = _NUM_RE.findall(str(value or ""))
if len(nums) < 2:
return None
lat, lon = float(nums[0]), float(nums[1])
return (lat, lon) if -90 <= lat <= 90 and -180 <= lon <= 180 else None
# --------------------------------------------------------------------------
# 3. Build the graph
# --------------------------------------------------------------------------
def cell_id(lat: float, lon: float, precision: int = DEFAULT_CELL_PRECISION) -> str:
f = 10 ** precision
return f"cell:{math.floor(lat * f)}_{math.floor(lon * f)}@{precision}"
def build_graph(docs: Sequence[Dict[str, Any]],
precision: int = DEFAULT_CELL_PRECISION) -> nx.MultiDiGraph:
"""Build the knowledge graph from raw index documents.
Node kinds: Account, Video, Frame, Object, Concept, Place, Cell.
Edge relations: HAS_VIDEO, HAS_FRAME, NEXT, CONTAINS, DEPICTS, AT_PLACE,
IN_CELL, ADJACENT, CO_OCCURS, REVISITS.
"""
G = nx.MultiDiGraph(precision=precision)
frames = [normalize(d) for d in docs]
by_video: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
for f in frames:
acc = f"account:{f['account_id']}"
vid = f"video:{f['account_id']}/{f['video_id']}"
fid = f"frame:{f['id']}"
G.add_node(acc, kind="Account", name=f["account_id"])
G.add_node(vid, kind="Video", name=f["video_id"], title=f["title"])
G.add_node(fid, kind="Frame", name=f["id"], video=f["video_id"],
frame_number=f["frame_number"], time=f["time"], caption=f["caption"],
dense_captions=f["dense_captions"], boundingbox=f["boundingbox"],
path=f["path"], lat=(f["latlon"] or (None, None))[0],
lon=(f["latlon"] or (None, None))[1], acl_users=f["acl_users"])
_edge(G, acc, vid, "HAS_VIDEO")
_edge(G, vid, fid, "HAS_FRAME")
by_video[vid].append(f)
for name, count in Counter(f["objects"]).items():
_edge(G, fid, _entity(G, "Object", name), "CONTAINS", count=count)
for name in set(f["tags"]) | set(f["labels"]):
src = "label" if name in f["labels"] else "tag"
_edge(G, fid, _entity(G, "Concept", name), "DEPICTS", source=src)
if f["location"]:
_edge(G, fid, _entity(G, "Place", f["location"]), "AT_PLACE")
if f["latlon"]:
cid = cell_id(*f["latlon"], precision)
G.add_node(cid, kind="Cell", name=cid, lat=f["latlon"][0], lon=f["latlon"][1])
_edge(G, fid, cid, "IN_CELL")
for a, b in combinations(sorted(set(f["objects"])), 2):
_bump(G, _entity(G, "Object", a), _entity(G, "Object", b), "CO_OCCURS")
_link_sequences(G, by_video)
_link_adjacent_cells(G, precision)
_link_revisits(G)
return G
def _entity(G: nx.MultiDiGraph, kind: str, name: str) -> str:
node = f"{kind.lower()}:{name}"
if node not in G:
G.add_node(node, kind=kind, name=name)
return node
def _edge(G: nx.MultiDiGraph, u: str, v: str, rel: str, **attrs: Any) -> None:
G.add_edge(u, v, key=rel, rel=rel, **attrs)
def _bump(G: nx.MultiDiGraph, u: str, v: str, rel: str) -> None:
if G.has_edge(u, v, key=rel):
G[u][v][rel]["weight"] += 1
else:
_edge(G, u, v, rel, weight=1)
def _order_key(f: Dict[str, Any]) -> Tuple[float, float]:
return (f["time"] if f["time"] is not None else math.inf,
f["frame_number"] if f["frame_number"] is not None else math.inf)
def _link_sequences(G: nx.MultiDiGraph, by_video: Dict[str, List[Dict[str, Any]]]) -> None:
"""NEXT edges: stored ``next_id`` when present, else frame/time order."""
for frames in by_video.values():
ids = {f["id"] for f in frames}
ordered = sorted(frames, key=_order_key)
for i, f in enumerate(ordered):
nxt = f["next_id"] if f["next_id"] in ids else (
ordered[i + 1]["id"] if i + 1 < len(ordered) else None)
if nxt:
g = next(x for x in frames if x["id"] == nxt)
gap = (g["time"] - f["time"]) if None not in (f["time"], g["time"]) else None
_edge(G, f"frame:{f['id']}", f"frame:{nxt}", "NEXT",
seconds=gap, frames=(g["frame_number"] or 0) - (f["frame_number"] or 0))
def _link_adjacent_cells(G: nx.MultiDiGraph, precision: int) -> None:
cells = {n for n, k in G.nodes(data="kind") if k == "Cell"}
for c in cells:
lat_i, lon_i = map(int, c[5:].split("@")[0].split("_"))
for dlat in (-1, 0, 1):
for dlon in (-1, 0, 1):
other = f"cell:{lat_i + dlat}_{lon_i + dlon}@{precision}"
if other != c and other in cells:
_edge(G, c, other, "ADJACENT")
def _link_revisits(G: nx.MultiDiGraph) -> None:
"""REVISITS: a later frame of the same cell, from another flight or after a gap."""
for cell in [n for n, k in G.nodes(data="kind") if k == "Cell"]:
visits = sorted((G.nodes[u] for u, _, r in G.in_edges(cell, keys=True) if r == "IN_CELL"),
key=lambda n: (n["time"] if n["time"] is not None else math.inf,
n["frame_number"] or 0))
for first, later in zip(visits, visits[1:]):
other_flight = first["video"] != later["video"]
if first["time"] is not None and later["time"] is not None:
gap = later["time"] - first["time"]
else:
gap = (later["frame_number"] or 0) - (first["frame_number"] or 0)
if other_flight or gap >= REVISIT_MIN_GAP:
unit = "s" if first["time"] is not None and later["time"] is not None else "frames"
_edge(G, f"frame:{later['name']}", f"frame:{first['name']}", "REVISITS",
cell=cell, gap=gap, gap_unit=unit, other_flight=other_flight)
# --------------------------------------------------------------------------
# 4. Graph-explorer style queries
# --------------------------------------------------------------------------
def _frames_with(G: nx.MultiDiGraph, node: str, rel: str) -> List[str]:
return [u for u, _, r in G.in_edges(node, keys=True) if r == rel] if node in G else []
def _sort_frames(G: nx.MultiDiGraph, frames: Iterable[str]) -> List[str]:
return sorted(frames, key=lambda f: (G.nodes[f]["time"] or math.inf,
G.nodes[f]["video"], G.nodes[f]["frame_number"] or 0))
def object_timeline(G: nx.MultiDiGraph, name: str) -> List[Dict[str, Any]]:
"""Every frame where an object (or a concept, as a fallback) appears, in order."""
name = name.lower()
hits = _frames_with(G, f"object:{name}", "CONTAINS") or _frames_with(G, f"concept:{name}", "DEPICTS")
rows = []
for f in _sort_frames(G, hits):
n = G.nodes[f]
count = G.get_edge_data(f, f"object:{name}", "CONTAINS", {}).get("count", 1)
rows.append({"frame": n["name"], "video": n["video"], "frame_number": n["frame_number"],
"time": n["time"], "lat": n["lat"], "lon": n["lon"], "count": count,
"caption": n["caption"]})
return rows
def frames_near(G: nx.MultiDiGraph, lat: float, lon: float, rings: int = 1,
start: Optional[float] = None, end: Optional[float] = None) -> List[str]:
"""Frames within ``rings`` cells of a point, optionally inside a time window."""
p = G.graph["precision"]
f = 10 ** p
lat_i, lon_i = math.floor(lat * f), math.floor(lon * f)
hits = set()
for dlat in range(-rings, rings + 1):
for dlon in range(-rings, rings + 1):
hits.update(_frames_with(G, f"cell:{lat_i + dlat}_{lon_i + dlon}@{p}", "IN_CELL"))
def in_window(fr: str) -> bool:
t = G.nodes[fr]["time"]
return t is None or ((start is None or t >= start) and (end is None or t <= end))
return _sort_frames(G, (h for h in hits if in_window(h)))
def revisited_places(G: nx.MultiDiGraph, min_visits: int = 2) -> List[Dict[str, Any]]:
"""Cells seen by several frames, with which videos saw them."""
rows = []
for c, kind in G.nodes(data="kind"):
if kind != "Cell":
continue
frames = _frames_with(G, c, "IN_CELL")
if len(frames) >= min_visits:
rows.append({"cell": c, "lat": G.nodes[c]["lat"], "lon": G.nodes[c]["lon"],
"frames": len(frames),
"videos": sorted({G.nodes[f]["video"] for f in frames})})
return sorted(rows, key=lambda r: (-len(r["videos"]), -r["frames"]))
def co_occurring(G: nx.MultiDiGraph, name: str, top: int = 5) -> List[Tuple[str, int]]:
"""Objects most often seen in the same frame as ``name``."""
node = f"object:{name.lower()}"
if node not in G:
return []
counts: Counter = Counter()
for u, v, r, d in list(G.out_edges(node, keys=True, data=True)) + \
list(G.in_edges(node, keys=True, data=True)):
if r == "CO_OCCURS":
counts[G.nodes[v if u == node else u]["name"]] += d["weight"]
return counts.most_common(top)
def spatiotemporal_context(G: nx.MultiDiGraph, frame_ids: Sequence[str],
max_lines: int = 40) -> str:
"""Context block for the chat agents, grown from frames search already found.
For each retrieved frame it adds: what it contains, the frames just before
and after it, and earlier frames of the same place (revisits). Pass the
ids the connected search agent returned; prepend the result to the prompt
alongside the ``[Context: account_id=..., video_id=...]`` line.
"""
lines: List[str] = []
for raw in frame_ids:
f = raw if raw.startswith("frame:") else f"frame:{raw}"
if f not in G:
continue
n = G.nodes[f]
objs = [f"{G.nodes[v]['name']} x{d.get('count', 1)}"
for _, v, r, d in G.out_edges(f, keys=True, data=True) if r == "CONTAINS"]
where = f" at ({n['lat']:.5f}, {n['lon']:.5f})" if n["lat"] is not None else ""
lines.append(f"Frame {n['name']} (video {n['video']}, #{n['frame_number']}"
f"{_when(n)}){where}: {n['caption']}"
+ (f" | objects: {', '.join(objs)}" if objs else ""))
before, after, revisits = [], [], []
for u, v, r, d in list(G.in_edges(f, keys=True, data=True)) + \
list(G.out_edges(f, keys=True, data=True)):
if r == "NEXT":
other = G.nodes[u if v == f else v]
(before if v == f else after).append(
f" {'before' if v == f else 'after'}: #{other['frame_number']}"
f"{_when(other)} {other['caption']}")
elif r == "REVISITS":
other = G.nodes[v if u == f else u]
kind = "another flight" if d["other_flight"] else f"{d['gap']:.0f} {d['gap_unit']} apart"
revisits.append(f" same place, {'earlier' if u == f else 'later'} ({kind}): "
f"{other['name']}{_when(other)} {other['caption']}")
lines.extend(before + after + revisits)
if len(lines) >= max_lines:
break
return "\n".join(lines[:max_lines])
def _when(n: Dict[str, Any]) -> str:
return (", " + datetime.fromtimestamp(n["time"], timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ")
if n.get("time") is not None else "")
# --------------------------------------------------------------------------
# 5. Exports for graph explorers
# --------------------------------------------------------------------------
def _flat(attrs: Dict[str, Any]) -> Dict[str, Any]:
"""GraphML/Cypher-safe attributes: drop None, join lists."""
out = {}
for k, v in attrs.items():
if v is None:
continue
out[k] = "; ".join(map(str, v)) if isinstance(v, (list, tuple)) else v
return out
def export_graph(G: nx.MultiDiGraph, out_dir: str) -> Dict[str, str]:
os.makedirs(out_dir, exist_ok=True)
flat = nx.MultiDiGraph()
flat.add_nodes_from((n, _flat(d)) for n, d in G.nodes(data=True))
flat.add_edges_from((u, v, k, _flat(d)) for u, v, k, d in G.edges(keys=True, data=True))
paths = {"graphml": os.path.join(out_dir, "dvsa_kg.graphml"),
"json": os.path.join(out_dir, "dvsa_kg.json"),
"cypher": os.path.join(out_dir, "dvsa_kg.cypher")}
nx.write_graphml(flat, paths["graphml"])
with open(paths["json"], "w", encoding="utf-8") as fh:
json.dump(nx.node_link_data(flat, edges="edges"), fh, indent=1)
with open(paths["cypher"], "w", encoding="utf-8") as fh:
fh.write(to_cypher(flat))
return paths
def to_cypher(G: nx.MultiDiGraph) -> str:
"""openCypher MERGE script (Neo4j Browser; Apache AGE wraps it in cypher())."""
def lit(v: Any) -> str:
if isinstance(v, bool):
return "true" if v else "false"
if isinstance(v, (int, float)):
return repr(v)
return "'" + str(v).replace("\\", "\\\\").replace("'", "\\'") + "'"
def props(d: Dict[str, Any]) -> str:
return ", ".join(f"{k}: {lit(v)}" for k, v in d.items() if k != "kind")
out = []
for n, d in G.nodes(data=True):
out.append(f"MERGE (n:{d['kind']} {{key: {lit(n)}}}) SET n += {{{props(d)}}};")
for u, v, k, d in G.edges(keys=True, data=True):
extra = {x: y for x, y in d.items() if x != "rel"}
out.append(f"MATCH (a {{key: {lit(u)}}}), (b {{key: {lit(v)}}}) "
f"MERGE (a)-[r:{k}]->(b)" + (f" SET r += {{{props(extra)}}}" if extra else "") + ";")
return "\n".join(out) + "\n"
# --------------------------------------------------------------------------
# Demo data shaped exactly like build_document() output
# --------------------------------------------------------------------------
def demo_documents() -> List[Dict[str, Any]]:
"""Two short flights over the same parking lot, as the indexer stores them."""
base = "https://sadronevideo.blob.core.windows.net/input/2/images/{v}/frame{n}.jpg?sv=redacted"
shots = [ # video, frame, minute, lat, lon, caption, objects, tags
("17", 0, 0, 47.67412, -122.12105, "a parking lot with cars",
["car", "car", "tree"], ["parking", "outdoor"]),
("17", 30, 1, 47.67431, -122.12088, "cars parked near a building",
["car", "building"], ["parking", "building"]),
("17", 60, 2, 47.67502, -122.12011, "a truck on a road",
["truck", "road"], ["road", "outdoor"]),
("17", 90, 3, 47.67588, -122.11940, "an empty field next to trees",
["tree"], ["field", "grass"]),
("21", 0, 120, 47.67415, -122.12101, "a nearly empty parking lot",
["car", "tree"], ["parking", "outdoor"]),
("21", 30, 121, 47.67505, -122.12015, "a truck and a person on a road",
["truck", "person", "road"], ["road"]),
]
docs = []
for v, n, minute, lat, lon, cap, objs, tags in shots:
docs.append({
"id": f"2-{n:04d}" if v == "17" else f"2-{1000 + n:04d}",
"account_id": "2",
"description": json.dumps({"caption": cap, "tags": tags, "objects": objs,
"dense_captions": []}),
"caption": cap, "labels": [], "tags": [], "objects": "",
"boundingbox": "0,0,1280,720",
"geotags": str((str(lat), str(lon))),
"path": base.format(v=v, n=n),
"timestamp": f"2026-11-01T10:{minute % 60:02d}:00+00:00" if minute < 60
else f"2026-11-01T12:{minute % 60:02d}:00+00:00",
"user": "ravi", "session": "demo",
})
return docs
# --------------------------------------------------------------------------
# CLI
# --------------------------------------------------------------------------
def main(argv: Optional[Sequence[str]] = None) -> None:
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
src = p.add_mutually_exclusive_group(required=True)
src.add_argument("--demo", action="store_true", help="use built-in sample documents")
src.add_argument("--from-json", help="JSON file with a list of index documents")
src.add_argument("--index", help="Azure AI Search index name, e.g. dvsa-index-s-2")
p.add_argument("--account-id")
p.add_argument("--video-id")
p.add_argument("--precision", type=int, default=DEFAULT_CELL_PRECISION)
p.add_argument("--object", default="car", help="object for the timeline query")
p.add_argument("--out", default="out/kg")
a = p.parse_args(argv)
if a.demo:
docs = demo_documents()
elif a.from_json:
with open(a.from_json, encoding="utf-8") as fh:
docs = json.load(fh)
else:
docs = fetch_index_documents(search_client_from_env(a.index), a.account_id, a.video_id)
G = build_graph(docs, precision=a.precision)
kinds = Counter(k for _, k in G.nodes(data="kind"))
rels = Counter(k for _, _, k in G.edges(keys=True))
print(f"{len(docs)} documents -> {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
print(" nodes:", dict(kinds))
print(" edges:", dict(rels))
print(f"\nTimeline of '{a.object}':")
for r in object_timeline(G, a.object):
print(f" video {r['video']} #{r['frame_number']} x{r['count']}: {r['caption']}")
print(f"\nCo-occurs with '{a.object}':", co_occurring(G, a.object))
print("\nRevisited places:")
for r in revisited_places(G)[:5]:
print(f" {r['cell']} seen in {r['frames']} frames by videos {r['videos']}")
first = object_timeline(G, a.object)
if first:
print("\nAgent context for", first[0]["frame"] + ":")
print(spatiotemporal_context(G, [first[0]["frame"]]))
print("\nWrote:", export_graph(G, a.out))
if __name__ == "__main__":
main()