Add fields to vector

2023-09-30 12:40:03 +08:00 · 2023-09-30 12:40:03 +08:00 · cdf5fcbfd8
parent 5916468ba7
commit cdf5fcbfd8
6 changed files with 95 additions and 32 deletions
--- a/object/store.go
+++ b/object/store.go
@ -197,6 +197,6 @@ func RefreshStoreVectors(store *Store) (bool, error) {
 		return false, err
 	}

-	ok, err := addVectorsForStore(storageProviderObj, embeddingProviderObj, "", store.Name)
+	ok, err := addVectorsForStore(storageProviderObj, embeddingProviderObj, "", store.Name, embeddingProvider.Name)
 	return ok, err
 }
--- a/object/vector.go
+++ b/object/vector.go
@ -26,11 +26,15 @@ type Vector struct {
 	Name        string `xorm:"varchar(100) notnull pk" json:"name"`
 	CreatedTime string `xorm:"varchar(100)" json:"createdTime"`

-	DisplayName string    `xorm:"varchar(100)" json:"displayName"`
-	Store       string    `xorm:"varchar(100)" json:"store"`
-	File        string    `xorm:"varchar(100)" json:"file"`
-	Text        string    `xorm:"mediumtext" json:"text"`
-	Data        []float32 `xorm:"mediumtext" json:"data"`
+	DisplayName string `xorm:"varchar(100)" json:"displayName"`
+	Store       string `xorm:"varchar(100)" json:"store"`
+	Provider    string `xorm:"varchar(100)" json:"provider"`
+	File        string `xorm:"varchar(100)" json:"file"`
+	Index       int    `json:"index"`
+	Text        string `xorm:"mediumtext" json:"text"`
+
+	Data      []float32 `xorm:"mediumtext" json:"data"`
+	Dimension int       `json:"dimension"`
 }

 func GetGlobalVectors() ([]*Vector, error) {
--- a/object/vector_embedding.go
+++ b/object/vector_embedding.go
@ -44,7 +44,7 @@ func filterTextFiles(files []*storage.Object) []*storage.Object {
 	return res
 }

-func addEmbeddedVector(embeddingProviderObj embedding.EmbeddingProvider, text string, storeName string, fileName string) (bool, error) {
+func addEmbeddedVector(embeddingProviderObj embedding.EmbeddingProvider, text string, storeName string, fileName string, index int, embeddingProviderName string) (bool, error) {
 	data, err := queryVectorSafe(embeddingProviderObj, text)
 	if err != nil {
 		return false, err
@ -61,14 +61,17 @@ func addEmbeddedVector(embeddingProviderObj embedding.EmbeddingProvider, text st
 		CreatedTime: util.GetCurrentTime(),
 		DisplayName: displayName,
 		Store:       storeName,
+		Provider:    embeddingProviderName,
 		File:        fileName,
+		Index:       index,
 		Text:        text,
 		Data:        data,
+		Dimension:   len(data),
 	}
 	return AddVector(vector)
 }

-func addVectorsForStore(storageProviderObj storage.StorageProvider, embeddingProviderObj embedding.EmbeddingProvider, prefix string, storeName string) (bool, error) {
+func addVectorsForStore(storageProviderObj storage.StorageProvider, embeddingProviderObj embedding.EmbeddingProvider, prefix string, storeName string, embeddingProviderName string) (bool, error) {
 	var affected bool

 	files, err := storageProviderObj.ListObjects(prefix)
@ -91,7 +94,7 @@ func addVectorsForStore(storageProviderObj storage.StorageProvider, embeddingPro
 		for i, textSection := range textSections {
 			if timeLimiter.Allow() {
 				fmt.Printf("[%d/%d] Generating embedding for store: [%s]'s text section: %s\n", i+1, len(textSections), storeName, textSection)
-				affected, err = addEmbeddedVector(embeddingProviderObj, textSection, storeName, file.Key)
+				affected, err = addEmbeddedVector(embeddingProviderObj, textSection, storeName, file.Key, i, embeddingProviderName)
 			} else {
 				err = timeLimiter.Wait(context.Background())
 				if err != nil {
@ -99,7 +102,7 @@ func addVectorsForStore(storageProviderObj storage.StorageProvider, embeddingPro
 				}

 				fmt.Printf("[%d/%d] Generating embedding for store: [%s]'s text section: %s\n", i+1, len(textSections), storeName, textSection)
-				affected, err = addEmbeddedVector(embeddingProviderObj, textSection, storeName, file.Key)
+				affected, err = addEmbeddedVector(embeddingProviderObj, textSection, storeName, file.Key, i, embeddingProviderName)
 			}
 		}
 	}
--- a/web/src/MessageListPage.js
+++ b/web/src/MessageListPage.js
@ -157,10 +157,14 @@ class MessageListPage extends React.Component {
        width: "100px",
        sorter: (a, b) => a.author.localeCompare(b.author),
        render: (text, record, index) => {
+          if (text === "AI") {
+            return text;
+          }
+
          return (
-            <Link to={`/member/${text}`}>
+            <a target="_blank" rel="noreferrer" href={Setting.getMyProfileUrl(this.props.account).replace("/account", `/users/${text}`)}>
              {text}
-            </Link>
+            </a>
          );
        },
      },
@ -168,14 +172,14 @@ class MessageListPage extends React.Component {
        title: i18next.t("message:Text"),
        dataIndex: "text",
        key: "text",
-        width: "150px",
+        width: "400px",
        sorter: (a, b) => a.text.localeCompare(b.text),
      },
      {
        title: i18next.t("general:Action"),
        dataIndex: "action",
        key: "action",
-        width: "130px",
+        width: "110px",
        render: (text, record, index) => {
          return (
            <div>
--- a/web/src/VectorEditPage.js
+++ b/web/src/VectorEditPage.js
@ -13,7 +13,7 @@
 // limitations under the License.

 import React from "react";
-import {Button, Card, Col, Input, Row} from "antd";
+import {Button, Card, Col, Input, InputNumber, Row} from "antd";
 import i18next from "i18next";
 import * as Setting from "./Setting";
 import * as VectorBackend from "./backend/VectorBackend";
@ -97,11 +97,21 @@ class VectorEditPage extends React.Component {
            {i18next.t("vector:Store")}:
          </Col>
          <Col span={22} >
-            <Input value={this.state.vector.store} onChange={e => {
+            <Input disabled={true} value={this.state.vector.store} onChange={e => {
              this.updateVectorField("store", e.target.value);
            }} />
          </Col>
        </Row>
+        <Row style={{marginTop: "20px"}} >
+          <Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
+            {i18next.t("vector:Provider")}:
+          </Col>
+          <Col span={22} >
+            <Input disabled={true} value={this.state.vector.provider} onChange={e => {
+              this.updateVectorField("provider", e.target.value);
+            }} />
+          </Col>
+        </Row>
        <Row style={{marginTop: "20px"}} >
          <Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
            {i18next.t("vector:File")}:
@ -122,6 +132,16 @@ class VectorEditPage extends React.Component {
            }} />
          </Col>
        </Row>
+        <Row style={{marginTop: "20px"}} >
+          <Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
+            {i18next.t("vector:Dimension")}:
+          </Col>
+          <Col span={22} >
+            <InputNumber disabled={true} value={this.state.vector.dimension} onChange={value => {
+              this.updateVectorField("dimension", value);
+            }} />
+          </Col>
+        </Row>
        <Row style={{marginTop: "20px"}} >
          <Col style={{marginTop: "5px"}} span={(Setting.isMobile()) ? 22 : 2}>
            {i18next.t("vector:Data")}:
--- a/web/src/VectorListPage.js
+++ b/web/src/VectorListPage.js
@ -14,7 +14,7 @@

 import React from "react";
 import {Link} from "react-router-dom";
-import {Button, Popconfirm, Table} from "antd";
+import {Button, Popconfirm, Table, Tooltip} from "antd";
 import moment from "moment";
 import * as Setting from "./Setting";
 import * as VectorBackend from "./backend/VectorBackend";
@ -111,18 +111,18 @@ class VectorListPage extends React.Component {
          );
        },
      },
-      {
-        title: i18next.t("general:Display name"),
-        dataIndex: "displayName",
-        key: "displayName",
-        width: "200px",
-        sorter: (a, b) => a.displayName.localeCompare(b.displayName),
-      },
+      // {
+      //   title: i18next.t("general:Display name"),
+      //   dataIndex: "displayName",
+      //   key: "displayName",
+      //   width: "200px",
+      //   sorter: (a, b) => a.displayName.localeCompare(b.displayName),
+      // },
      {
        title: i18next.t("vector:Store"),
        dataIndex: "store",
        key: "store",
-        width: "200px",
+        width: "130px",
        sorter: (a, b) => a.store.localeCompare(b.store),
        render: (text, record, index) => {
          return (
@ -132,6 +132,20 @@ class VectorListPage extends React.Component {
          );
        },
      },
+      {
+        title: i18next.t("vector:Provider"),
+        dataIndex: "provider",
+        key: "provider",
+        width: "200px",
+        sorter: (a, b) => a.provider.localeCompare(b.provider),
+        render: (text, record, index) => {
+          return (
+            <Link to={`/providers/${text}`}>
+              {text}
+            </Link>
+          );
+        },
+      },
      {
        title: i18next.t("vector:File"),
        dataIndex: "file",
@ -139,6 +153,13 @@ class VectorListPage extends React.Component {
        width: "200px",
        sorter: (a, b) => a.file.localeCompare(b.file),
      },
+      {
+        title: i18next.t("vector:Index"),
+        dataIndex: "index",
+        key: "index",
+        width: "80px",
+        sorter: (a, b) => a.index - b.index,
+      },
      {
        title: i18next.t("vector:Text"),
        dataIndex: "text",
@ -147,9 +168,11 @@ class VectorListPage extends React.Component {
        sorter: (a, b) => a.text.localeCompare(b.text),
        render: (text, record, index) => {
          return (
-            <div style={{maxWidth: "200px"}}>
-              {Setting.getShortText(text)}
-            </div>
+            <Tooltip placement="left" title={text}>
+              <div style={{maxWidth: "200px"}}>
+                {Setting.getShortText(text)}
+              </div>
+            </Tooltip>
          );
        },
      },
@ -161,17 +184,26 @@ class VectorListPage extends React.Component {
        sorter: (a, b) => a.data.localeCompare(b.data),
        render: (text, record, index) => {
          return (
-            <div style={{maxWidth: "200px"}}>
-              {Setting.getShortText(JSON.stringify(text))}
-            </div>
+            <Tooltip placement="left" title={Setting.getShortText(JSON.stringify(text), 1000)}>
+              <div style={{maxWidth: "200px"}}>
+                {Setting.getShortText(JSON.stringify(text))}
+              </div>
+            </Tooltip>
          );
        },
      },
+      {
+        title: i18next.t("vector:Dimension"),
+        dataIndex: "dimension",
+        key: "dimension",
+        width: "80px",
+        sorter: (a, b) => a.dimension - b.dimension,
+      },
      {
        title: i18next.t("general:Action"),
        dataIndex: "action",
        key: "action",
-        width: "180px",
+        width: "150px",
        render: (text, record, index) => {
          return (
            <div>