mirror of https://github.com/0xplaygrounds/rig
refactor(lancedb): docs, make examples idempotent
refactor: clippy
This commit is contained in:
parent
92c91d23c3
commit
5ed3fcc401
|
@ -17,6 +17,9 @@
|
||||||
## Rig-Lancedb
|
## Rig-Lancedb
|
||||||
This companion crate implements a Rig vector store based on Lancedb.
|
This companion crate implements a Rig vector store based on Lancedb.
|
||||||
|
|
||||||
|
## Pre-requisites
|
||||||
|
If you are using `rig-lancedb` locally, you must ensure you have `protoc` (the [Protobuf Compiler](https://protobuf.dev/installation/)).
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
Add the companion crate to your `Cargo.toml`, along with the rig-core crate:
|
Add the companion crate to your `Cargo.toml`, along with the rig-core crate:
|
||||||
|
|
|
@ -38,8 +38,15 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
.build()
|
.build()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let table = db
|
let table = if db
|
||||||
.create_table(
|
.table_names()
|
||||||
|
.execute()
|
||||||
|
.await?
|
||||||
|
.contains(&"definitions".to_string())
|
||||||
|
{
|
||||||
|
db.open_table("definitions").execute().await?
|
||||||
|
} else {
|
||||||
|
db.create_table(
|
||||||
"definitions",
|
"definitions",
|
||||||
RecordBatchIterator::new(
|
RecordBatchIterator::new(
|
||||||
vec![as_record_batch(embeddings, model.ndims())],
|
vec![as_record_batch(embeddings, model.ndims())],
|
||||||
|
@ -47,16 +54,19 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.execute()
|
.execute()
|
||||||
.await?;
|
.await?
|
||||||
|
};
|
||||||
|
|
||||||
// See [LanceDB indexing](https://lancedb.github.io/lancedb/concepts/index_ivfpq/#product-quantization) for more information
|
// See [LanceDB indexing](https://lancedb.github.io/lancedb/concepts/index_ivfpq/#product-quantization) for more information
|
||||||
table
|
if table.index_stats("embedding").await?.is_none() {
|
||||||
.create_index(
|
table
|
||||||
&["embedding"],
|
.create_index(
|
||||||
lancedb::index::Index::IvfPq(IvfPqIndexBuilder::default()),
|
&["embedding"],
|
||||||
)
|
lancedb::index::Index::IvfPq(IvfPqIndexBuilder::default()),
|
||||||
.execute()
|
)
|
||||||
.await?;
|
.execute()
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
// Define search_params params that will be used by the vector store to perform the vector search.
|
// Define search_params params that will be used by the vector store to perform the vector search.
|
||||||
let search_params = SearchParams::default();
|
let search_params = SearchParams::default();
|
||||||
|
|
|
@ -32,8 +32,15 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
// Initialize LanceDB locally.
|
// Initialize LanceDB locally.
|
||||||
let db = lancedb::connect("data/lancedb-store").execute().await?;
|
let db = lancedb::connect("data/lancedb-store").execute().await?;
|
||||||
|
|
||||||
let table = db
|
let table = if db
|
||||||
.create_table(
|
.table_names()
|
||||||
|
.execute()
|
||||||
|
.await?
|
||||||
|
.contains(&"definitions".to_string())
|
||||||
|
{
|
||||||
|
db.open_table("definitions").execute().await?
|
||||||
|
} else {
|
||||||
|
db.create_table(
|
||||||
"definitions",
|
"definitions",
|
||||||
RecordBatchIterator::new(
|
RecordBatchIterator::new(
|
||||||
vec![as_record_batch(embeddings, model.ndims())],
|
vec![as_record_batch(embeddings, model.ndims())],
|
||||||
|
@ -41,7 +48,8 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.execute()
|
.execute()
|
||||||
.await?;
|
.await?
|
||||||
|
};
|
||||||
|
|
||||||
let vector_store = LanceDbVectorIndex::new(table, model, "id", search_params).await?;
|
let vector_store = LanceDbVectorIndex::new(table, model, "id", search_params).await?;
|
||||||
|
|
||||||
|
|
|
@ -44,8 +44,15 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
.build()
|
.build()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let table = db
|
let table = if db
|
||||||
.create_table(
|
.table_names()
|
||||||
|
.execute()
|
||||||
|
.await?
|
||||||
|
.contains(&"definitions".to_string())
|
||||||
|
{
|
||||||
|
db.open_table("definitions").execute().await?
|
||||||
|
} else {
|
||||||
|
db.create_table(
|
||||||
"definitions",
|
"definitions",
|
||||||
RecordBatchIterator::new(
|
RecordBatchIterator::new(
|
||||||
vec![as_record_batch(embeddings, model.ndims())],
|
vec![as_record_batch(embeddings, model.ndims())],
|
||||||
|
@ -53,21 +60,24 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.execute()
|
.execute()
|
||||||
.await?;
|
.await?
|
||||||
|
};
|
||||||
|
|
||||||
// See [LanceDB indexing](https://lancedb.github.io/lancedb/concepts/index_ivfpq/#product-quantization) for more information
|
// See [LanceDB indexing](https://lancedb.github.io/lancedb/concepts/index_ivfpq/#product-quantization) for more information
|
||||||
table
|
if table.index_stats("embedding").await?.is_none() {
|
||||||
.create_index(
|
table
|
||||||
&["embedding"],
|
.create_index(
|
||||||
lancedb::index::Index::IvfPq(
|
&["embedding"],
|
||||||
IvfPqIndexBuilder::default()
|
lancedb::index::Index::IvfPq(
|
||||||
// This overrides the default distance type of L2.
|
IvfPqIndexBuilder::default()
|
||||||
// Needs to be the same distance type as the one used in search params.
|
// This overrides the default distance type of L2.
|
||||||
.distance_type(DistanceType::Cosine),
|
// Needs to be the same distance type as the one used in search params.
|
||||||
),
|
.distance_type(DistanceType::Cosine),
|
||||||
)
|
),
|
||||||
.execute()
|
)
|
||||||
.await?;
|
.execute()
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
// Define search_params params that will be used by the vector store to perform the vector search.
|
// Define search_params params that will be used by the vector store to perform the vector search.
|
||||||
let search_params = SearchParams::default().distance_type(DistanceType::Cosine);
|
let search_params = SearchParams::default().distance_type(DistanceType::Cosine);
|
||||||
|
|
Loading…
Reference in New Issue