Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/snippets/multimodal.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ export const PyBlobApiSchema = "import pyarrow as pa\n\n# Define schema with Blo

export const PyBlobApiToPandas = "# Default: blob columns come back lazily\ndf_lazy = tbl.to_pandas()\n\n# Materialize blob bytes eagerly\ndf_bytes = tbl.to_pandas(blob_mode=\"bytes\")\n\n# Return descriptors instead of payloads\ndf_desc = tbl.to_pandas(blob_mode=\"descriptions\")\n\n# Forward extra kwargs to PyArrow's to_pandas\ndf_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)\n";

export const PyBlobUriWrite = "# Declare a blob column with the Lance blob extension type\nschema = pa.schema([pa.field(\"id\", pa.int64()), lancedb.blob(\"image\")])\ntbl = db.create_table(\"images_by_uri\", schema=schema)\n\n# Point at an existing object; file://, s3://, and other URIs work\nimage_uri = blob_path.as_uri()\n\n# A string value coerces to the blob's `uri` field, so the row stores\n# a reference instead of copying the bytes into the table\ntbl.add(\n [{\"id\": 1, \"image\": image_uri}],\n allow_external_blob_outside_bases=True,\n)\n\n# Fetch the referenced bytes back through the blob column\nhits = tbl.search().to_arrow()\nblobs = tbl.fetch_blobs(\"image\", hits)\n";

export const PyCreateDummyData = "# Create some dummy images\ndef create_dummy_image(color):\n img = Image.new('RGB', (100, 100), color=color)\n buf = io.BytesIO()\n img.save(buf, format='PNG')\n return buf.getvalue()\n\n# Create dataset with metadata, vectors, and image blobs\ndata = [\n {\n \"id\": 1,\n \"filename\": \"red_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('red'),\n \"label\": \"red\"\n },\n {\n \"id\": 2,\n \"filename\": \"blue_square.png\",\n \"vector\": np.random.rand(128).astype(np.float32),\n \"image_blob\": create_dummy_image('blue'),\n \"label\": \"blue\"\n }\n]\n";

export const PyDefineSchema = "# Define schema explictly to ensure image_blob is treated as binary\nschema = pa.schema([\n pa.field(\"id\", pa.int32()),\n pa.field(\"filename\", pa.string()),\n pa.field(\"vector\", pa.list_(pa.float32(), 128)),\n pa.field(\"image_blob\", pa.binary()), # Important: Use pa.binary() for blobs\n pa.field(\"label\", pa.string())\n])\n";
Expand Down
29 changes: 29 additions & 0 deletions docs/tables/multimodal.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import {
RsBlobApiIngest as RsBlobApiIngest,
PyBlobApiToPandas as BlobApiToPandas,
PyQueryToPandasKwargs as QueryToPandasKwargs,
PyBlobUriWrite as BlobUriWrite,
} from '/snippets/multimodal.mdx';

LanceDB handles multimodal data—images, audio, video, and PDF files—natively by storing the raw bytes in a binary column alongside your vectors and metadata. This approach simplifies your data infrastructure by keeping the raw assets and their embeddings in the same database, eliminating the need for separate object storage for many use cases.
Expand Down Expand Up @@ -228,6 +229,34 @@ Query builders also accept `blob_mode` on their `to_pandas()` method:
</CodeBlock>
</CodeGroup>

## Write blob URIs

Copying bytes into the table is not always practical when your media already lives in object storage. A blob column can instead store a reference to the original object.

Declare the column with `lancedb.blob()` and pass a string URI as the value in `add()`. LanceDB coerces the string to the blob's `uri` field, so the row stores a reference instead of a copy. `fetch_blobs()` reads the payload from the referenced location.

How `add()` handles a URI depends on where it points:

- A URI under a registered blob base writes with no extra options.
- A URI outside every registered base makes `add()` fail.
- `allow_external_blob_outside_bases=True` is an escape hatch that stores the absolute URI anyway. It does not register a base.

<CodeGroup>
<CodeBlock filename="Python" language="Python" icon="python">
{BlobUriWrite}
</CodeBlock>
</CodeGroup>

<Warning>
A row written with `allow_external_blob_outside_bases=True` keeps only a reference. The object has to stay readable at that URI for `fetch_blobs()` to work.
</Warning>

The flag applies to local (OSS) tables only. Enterprise tables reject `allow_external_blob_outside_bases` before making a request, although string URIs still coerce and are sent as a `uri` struct. In Rust, set the flag with the `allow_external_blob_outside_bases` method on the `add` builder.

<Note>
URI coercion applies to `add()` only. `merge_insert` does not coerce string blob input.
</Note>

## Other modalities

The `pa.binary()` and `pa.large_binary()` types are universal. You can use this same pattern for other types of multimodal data:
Expand Down
30 changes: 30 additions & 0 deletions tests/py/test_multimodal.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,3 +230,33 @@ async def test_query_to_pandas_kwargs(db_path_factory):
assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
assert len(df_vec) == 10
assert "video" not in df_vec.columns


def test_blob_uri_write(db_path_factory, tmp_path):
payload = b"fake_image_bytes"
blob_path = tmp_path / "cat.jpg"
blob_path.write_bytes(payload)

db = lancedb.connect(db_path_factory("blob_uri_db"))

# --8<-- [start:blob_uri_write]
# Declare a blob column with the Lance blob extension type
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
tbl = db.create_table("images_by_uri", schema=schema)

# Point at an existing object; file://, s3://, and other URIs work
image_uri = blob_path.as_uri()

# A string value coerces to the blob's `uri` field, so the row stores
# a reference instead of copying the bytes into the table
tbl.add(
[{"id": 1, "image": image_uri}],
allow_external_blob_outside_bases=True,
)

# Fetch the referenced bytes back through the blob column
hits = tbl.search().to_arrow()
blobs = tbl.fetch_blobs("image", hits)
# --8<-- [end:blob_uri_write]

assert blobs[0].as_py() == payload
Loading