dnrops
/
demo-code-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

post-processing and highlight in ui

This commit is contained in:
Andrey Vasnetsov 2023-02-18 19:29:45 +01:00
parent 86fb5d66a1
commit ef9c2408e7
11 changed files with 356 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
code_search/config.py
View File

@ -11,5 +11,8 @@ DATA_DIR = os.path.join(ROOT_DIR, "data")
QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
QDRANT_COLLECTION_NAME = "code-snippets-unixcoder"
QDRANT_CODE_COLLECTION_NAME = "code-snippets-unixcoder"
QDRANT_NLU_COLLECTION_NAME = "code-signatures"
ENCODER_NAME = "all-MiniLM-L6-v2"
ENCODER_SIZE = 384

4
code_search/index/upload_code.py
View File

@ -6,7 +6,7 @@ import qdrant_client
import numpy as np
import json
from code_search.config import QDRANT_URL, QDRANT_API_KEY, DATA_DIR
from code_search.config import QDRANT_URL, QDRANT_API_KEY, DATA_DIR, QDRANT_CODE_COLLECTION_NAME
from code_search.model.encoder import UniXcoderEmbeddingsProvider
code_keys = [
@ -24,7 +24,7 @@ def encode_and_upload():
prefer_grpc=True,
)
collection_name = "code-snippets-unixcoder"
collection_name = QDRANT_CODE_COLLECTION_NAME
input_file = Path(DATA_DIR) / "qdrant_snippets.jsonl"
encoder = UniXcoderEmbeddingsProvider(device="cuda")

8
code_search/index/upload_signatures.py
View File

@ -6,14 +6,12 @@ from qdrant_client import QdrantClient
from qdrant_client.http.models import Distance, VectorParams
from sentence_transformers import SentenceTransformer
from code_search.config import DATA_DIR, QDRANT_URL, QDRANT_API_KEY
from code_search.config import DATA_DIR, QDRANT_URL, QDRANT_API_KEY, QDRANT_NLU_COLLECTION_NAME, ENCODER_NAME, \
ENCODER_SIZE
from code_search.index.textifier import textify
file_name = Path(DATA_DIR) / "structures.json"
ENCODER_NAME = "all-MiniLM-L6-v2"
ENCODER_SIZE = 384
def iter_batch(iterable, batch_size=64):
batch = []
@ -42,7 +40,7 @@ def encode(sentence_transformer_name=ENCODER_NAME):
def upload():
collection_name = "code-signatures"
collection_name = QDRANT_NLU_COLLECTION_NAME
client = QdrantClient(
QDRANT_URL,

126
code_search/postprocessing.py Normal file
View File

@ -0,0 +1,126 @@
from collections import defaultdict
from typing import List
def merge_search_results(code_search_result: List[dict], nlu_search_result: List[dict]) -> List[dict]:
"""Merge search results from code and NLU searchers
Args:
code_search_result (List[dict]): Code search results
Examples:
[
{"end_line": 127, "file": "lib/segment/src/index/query_estimator.rs", "start_line": 123}
{"end_line": 830, "file": "lib/segment/src/segment.rs", "start_line": 827}
{"end_line": 169, "file": "lib/segment/src/index/field_index/field_index_base.rs", "start_line": 166}
{"end_line": 162, "file": "lib/segment/src/index/query_estimator.rs", "start_line": 158}
{"end_line": 152, "file": "lib/collection/src/shards/local_shard_operations.rs", "start_line": 150}
]
nlu_search_result (List[dict]): NLU search results
Examples:
[
{
"code_type": "Function",
"context": {
"file_name": "query_estimator.rs",
"file_path": "lib/segment/src/index/query_estimator.rs",
"module": "index",
"snippet": "...",
"struct_name": null
},
"docstring": null,
"line": 13,
"line_from": 13,
"line_to": 39,
"name": "combine_should_estimations",
"signature": "fn combine_should_estimations () -> CardinalityEstimation"
}
]
"""
code_search_result_by_file = defaultdict(list)
for hit in code_search_result:
code_search_result_by_file[hit["file"]].append(hit)
for nlu_search_hit in nlu_search_result:
file = nlu_search_hit["context"]["file_path"]
if file in code_search_result_by_file:
nlu_search_hit["sub_matches"] = try_merge_overlapping_snippets(
code_search_result_by_file[file],
nlu_search_hit
)
for sub_match in nlu_search_hit["sub_matches"]:
nlu_search_hit["context"]["snippet"] = highlight_overlap(
text=nlu_search_hit["context"]["snippet"],
start_line=nlu_search_hit["line_from"],
from_line=sub_match["overlap_from"],
to_line=sub_match["overlap_to"]
)
nlu_search_result = sorted(nlu_search_result, key=lambda x: -len(x.get('sub_matches', [])))
return nlu_search_result
def highlight_overlap(text: str, start_line: int, from_line: int, to_line: int, start_tag="<mark>", end_tag="</mark>"):
"""Highlight overlapping lines in the text using start_tag and end_tag
Args:
text:
start_line:
from_line:
to_line:
start_tag:
end_tag:
Returns: Highlighted text
"""
lines = text.split("\n")
highlighted_lines = []
for i, line in enumerate(lines):
if i + start_line in range(from_line, to_line + 1):
highlighted_lines.append(f"{start_tag}{line}{end_tag}")
else:
highlighted_lines.append(line)
return "\n".join(highlighted_lines)
def try_merge_overlapping_snippets(code_search_results: List[dict], nlu_search_result: dict) -> List[dict]:
"""Find code search results that overlap with NLU search results and merge them
Use nlu_search_result as a base for merging
Args:
code_search_results:
[
{"end_line": 127, "start_line": 123, ...}
{"end_line": 830, "start_line": 827, ...}
{"end_line": 169, "start_line": 166, ...}
{"end_line": 162, "start_line": 158, ...}
{"end_line": 14, "start_line": 16, ...}
]
nlu_search_result:
{
"line": 13,
"line_from": 13,
"line_to": 39,
...
}
Returns: Overlapping code search results merged with NLU search results
"""
overlapped = []
code_search_result = sorted(code_search_results, key=lambda x: x["start_line"])
for code_search_hit in code_search_result:
from_a = code_search_hit["start_line"] + 1
to_a = code_search_hit["end_line"] + 1
from_b = nlu_search_result["line_from"]
to_b = nlu_search_result["line_to"]
# get overlapping range
start = max(from_a, from_b)
end = min(to_a, to_b)
if start <= end:
overlapped.append({
"overlap_from": start,
"overlap_to": end,
})
return overlapped

51
code_search/searcher.py
View File

@ -2,16 +2,18 @@ import json
from typing import List
from qdrant_client import QdrantClient
from qdrant_client.models import ScoredPoint
from sentence_transformers import SentenceTransformer
from code_search.config import QDRANT_URL, QDRANT_API_KEY, QDRANT_COLLECTION_NAME
from code_search.config import QDRANT_URL, QDRANT_API_KEY, ENCODER_NAME, QDRANT_CODE_COLLECTION_NAME, \
QDRANT_NLU_COLLECTION_NAME
from code_search.model.encoder import UniXcoderEmbeddingsProvider
from code_search.postprocessing import merge_search_results
class CodeSearcher:
def __init__(self, collection_name):
self.collection_name = collection_name
def __init__(self):
self.collection_name = QDRANT_CODE_COLLECTION_NAME
self.client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
self.encoder = UniXcoderEmbeddingsProvider("cpu")
@ -21,13 +23,50 @@ class CodeSearcher:
collection_name=self.collection_name,
query_vector=vector,
limit=limit,
with_payload=["start_line", "end_line", "file"]
)
return [hit.payload for hit in result]
class NluSearcher:
def __init__(self):
self.collection_name = QDRANT_NLU_COLLECTION_NAME
self.client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
self.encoder = SentenceTransformer(ENCODER_NAME)
def search(self, query, limit=5) -> List[dict]:
vector = self.encoder.encode([query])[0].tolist()
result = self.client.search(
collection_name=self.collection_name,
query_vector=vector,
limit=limit,
)
return [hit.payload for hit in result]
class CombinedSearcher:
def __init__(self):
self.nlu_searcher = NluSearcher()
self.code_searcher = CodeSearcher()
def search(self, query, limit=5, code_limit=20) -> List[dict]:
nlu_res = self.nlu_searcher.search(query, limit=limit)
code_res = self.code_searcher.search(query, limit=code_limit)
merged_results = merge_search_results(code_res, nlu_res)
return merged_results
if __name__ == '__main__':
searcher = CodeSearcher(QDRANT_COLLECTION_NAME)
res = searcher.search("cardinality of should request")
query = "cardinality of should request"
searcher = CombinedSearcher()
res = searcher.search(query)
for hit in res:
print(json.dumps(hit))

6
code_search/service.py
View File

@ -3,12 +3,12 @@ import os
from fastapi import FastAPI
from starlette.staticfiles import StaticFiles
from code_search.config import ROOT_DIR, QDRANT_COLLECTION_NAME
from code_search.searcher import CodeSearcher
from code_search.config import ROOT_DIR
from code_search.searcher import CombinedSearcher
app = FastAPI()
searcher = CodeSearcher(QDRANT_COLLECTION_NAME)
searcher = CombinedSearcher()
@app.get("/api/search")

169
frontend/package-lock.json generated
View File

@ -15,7 +15,9 @@
"highlight.js": "^11.7.0",
"quasar": "^2.6.0",
"vue": "^3.0.0",
"vue-router": "^4.0.0"
"vue-code-highlight": "^0.7.8",
"vue-router": "^4.0.0",
"vue3-highlightjs": "^1.0.5"
},
"devDependencies": {
"@babel/eslint-parser": "^7.13.14",
@ -468,9 +470,9 @@
}
},
"node_modules/@babel/parser": {
"version": "7.17.9",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.9.tgz",
"integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg==",
"version": "7.20.15",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.20.15.tgz",
"integrity": "sha512-DI4a1oZuf8wC+oAJA9RW6ga3Zbe8RZFt7kD9i4qAspz3I/yHet1VvC3DiSy/fsUvv5pvJuNPh0LPOdCcqinDPg==",
"bin": {
"parser": "bin/babel-parser.js"
},
@ -7413,9 +7415,9 @@
"dev": true
},
"node_modules/nanoid": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz",
"integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==",
"version": "3.3.4",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz",
"integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==",
"bin": {
"nanoid": "bin/nanoid.cjs"
},
@ -8102,9 +8104,9 @@
}
},
"node_modules/postcss": {
"version": "8.4.12",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.12.tgz",
"integrity": "sha512-lg6eITwYe9v6Hr5CncVbK70SoioNQIq81nsaG86ev5hAidQvmOeETBqs7jm43K2F5/Ley3ytDtriImV6TpNiSg==",
"version": "8.4.21",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
"integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
"funding": [
{
"type": "opencollective",
@ -8116,7 +8118,7 @@
}
],
"dependencies": {
"nanoid": "^3.3.1",
"nanoid": "^3.3.4",
"picocolors": "^1.0.0",
"source-map-js": "^1.0.2"
},
@ -8720,6 +8722,11 @@
"renderkid": "^3.0.0"
}
},
"node_modules/prism-es6": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/prism-es6/-/prism-es6-1.2.0.tgz",
"integrity": "sha512-A8JV9G2zKM8PWksT7YJcmnaWtYO6C9hSfxM/xv0RxB2aNc8rjv30WakzIw1gWyqLi2eiqquo2KmS7orxqlm+yg=="
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -10422,6 +10429,47 @@
"@vue/shared": "3.2.33"
}
},
"node_modules/vue-code-highlight": {
"version": "0.7.8",
"resolved": "https://registry.npmjs.org/vue-code-highlight/-/vue-code-highlight-0.7.8.tgz",
"integrity": "sha512-jMt1W0DGisNZ3B2TASFCPLEswohVrwImX+TxzMIZINZyAUUNZ1Xth3soJw1xepTsMgxkIkhsmN/xVNpiI3bU4g==",
"dependencies": {
"prism-es6": "^1.2.0",
"vue": "^2.5.16"
}
},
"node_modules/vue-code-highlight/node_modules/@vue/compiler-sfc": {
"version": "2.7.14",
"resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-2.7.14.tgz",
"integrity": "sha512-aNmNHyLPsw+sVvlQFQ2/8sjNuLtK54TC6cuKnVzAY93ks4ZBrvwQSnkkIh7bsbNhum5hJBS00wSDipQ937f5DA==",
"dependencies": {
"@babel/parser": "^7.18.4",
"postcss": "^8.4.14",
"source-map": "^0.6.1"
}
},
"node_modules/vue-code-highlight/node_modules/csstype": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz",
"integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="
},
"node_modules/vue-code-highlight/node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/vue-code-highlight/node_modules/vue": {
"version": "2.7.14",
"resolved": "https://registry.npmjs.org/vue/-/vue-2.7.14.tgz",
"integrity": "sha512-b2qkFyOM0kwqWFuQmgd4o+uHGU7T+2z3T+WQp8UBjADfEv2n4FEMffzBmCKNP0IGzOEEfYjvtcC62xaSKeQDrQ==",
"dependencies": {
"@vue/compiler-sfc": "2.7.14",
"csstype": "^3.1.0"
}
},
"node_modules/vue-eslint-parser": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-8.3.0.tgz",
@ -10644,6 +10692,25 @@
"node": ">=4.0.0"
}
},
"node_modules/vue3-highlightjs": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/vue3-highlightjs/-/vue3-highlightjs-1.0.5.tgz",
"integrity": "sha512-Q4YNPXu0X5VMBnwPVOk+IQf1Ohp9jFdMitEAmzaz8qVVefcQpN6Dx4BnDGKxja3TLDVF+EgL136wC8YzmoCX9w==",
"dependencies": {
"highlight.js": "^10.3.2"
},
"peerDependencies": {
"vue": "^3.0.0"
}
},
"node_modules/vue3-highlightjs/node_modules/highlight.js": {
"version": "10.7.3",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
"integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==",
"engines": {
"node": "*"
}
},
"node_modules/watchpack": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.3.1.tgz",
@ -11637,9 +11704,9 @@
}
},
"@babel/parser": {
"version": "7.17.9",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.17.9.tgz",
"integrity": "sha512-vqUSBLP8dQHFPdPi9bc5GK9vRkYHJ49fsZdtoJ8EQ8ibpwk5rPKfvNIwChB0KVXcIjcepEBBd2VHC5r9Gy8ueg=="
"version": "7.20.15",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.20.15.tgz",
"integrity": "sha512-DI4a1oZuf8wC+oAJA9RW6ga3Zbe8RZFt7kD9i4qAspz3I/yHet1VvC3DiSy/fsUvv5pvJuNPh0LPOdCcqinDPg=="
},
"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": {
"version": "7.16.7",
@ -16827,9 +16894,9 @@
"dev": true
},
"nanoid": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz",
"integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w=="
"version": "3.3.4",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz",
"integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw=="
},
"natural-compare": {
"version": "1.4.0",
@ -17340,11 +17407,11 @@
}
},
"postcss": {
"version": "8.4.12",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.12.tgz",
"integrity": "sha512-lg6eITwYe9v6Hr5CncVbK70SoioNQIq81nsaG86ev5hAidQvmOeETBqs7jm43K2F5/Ley3ytDtriImV6TpNiSg==",
"version": "8.4.21",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
"integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
"requires": {
"nanoid": "^3.3.1",
"nanoid": "^3.3.4",
"picocolors": "^1.0.0",
"source-map-js": "^1.0.2"
}
@ -17726,6 +17793,11 @@
"renderkid": "^3.0.0"
}
},
"prism-es6": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/prism-es6/-/prism-es6-1.2.0.tgz",
"integrity": "sha512-A8JV9G2zKM8PWksT7YJcmnaWtYO6C9hSfxM/xv0RxB2aNc8rjv30WakzIw1gWyqLi2eiqquo2KmS7orxqlm+yg=="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -18977,6 +19049,46 @@
"@vue/shared": "3.2.33"
}
},
"vue-code-highlight": {
"version": "0.7.8",
"resolved": "https://registry.npmjs.org/vue-code-highlight/-/vue-code-highlight-0.7.8.tgz",
"integrity": "sha512-jMt1W0DGisNZ3B2TASFCPLEswohVrwImX+TxzMIZINZyAUUNZ1Xth3soJw1xepTsMgxkIkhsmN/xVNpiI3bU4g==",
"requires": {
"prism-es6": "^1.2.0",
"vue": "^2.5.16"
},
"dependencies": {
"@vue/compiler-sfc": {
"version": "2.7.14",
"resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-2.7.14.tgz",
"integrity": "sha512-aNmNHyLPsw+sVvlQFQ2/8sjNuLtK54TC6cuKnVzAY93ks4ZBrvwQSnkkIh7bsbNhum5hJBS00wSDipQ937f5DA==",
"requires": {
"@babel/parser": "^7.18.4",
"postcss": "^8.4.14",
"source-map": "^0.6.1"
}
},
"csstype": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz",
"integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="
},
"source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
},
"vue": {
"version": "2.7.14",
"resolved": "https://registry.npmjs.org/vue/-/vue-2.7.14.tgz",
"integrity": "sha512-b2qkFyOM0kwqWFuQmgd4o+uHGU7T+2z3T+WQp8UBjADfEv2n4FEMffzBmCKNP0IGzOEEfYjvtcC62xaSKeQDrQ==",
"requires": {
"@vue/compiler-sfc": "2.7.14",
"csstype": "^3.1.0"
}
}
}
},
"vue-eslint-parser": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-8.3.0.tgz",
@ -19142,6 +19254,21 @@
}
}
},
"vue3-highlightjs": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/vue3-highlightjs/-/vue3-highlightjs-1.0.5.tgz",
"integrity": "sha512-Q4YNPXu0X5VMBnwPVOk+IQf1Ohp9jFdMitEAmzaz8qVVefcQpN6Dx4BnDGKxja3TLDVF+EgL136wC8YzmoCX9w==",
"requires": {
"highlight.js": "^10.3.2"
},
"dependencies": {
"highlight.js": {
"version": "10.7.3",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
"integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A=="
}
}
},
"watchpack": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.3.1.tgz",

4
frontend/package.json
View File

@ -18,7 +18,9 @@
"highlight.js": "^11.7.0",
"quasar": "^2.6.0",
"vue": "^3.0.0",
"vue-router": "^4.0.0"
"vue-code-highlight": "^0.7.8",
"vue-router": "^4.0.0",
"vue3-highlightjs": "^1.0.5"
},
"devDependencies": {
"@babel/eslint-parser": "^7.13.14",

1
frontend/src/boot/highlight.js
View File

@ -2,6 +2,7 @@ import hljs from 'highlight.js/lib/core';
import rust from 'highlight.js/lib/languages/rust';
import hljsVuePlugin from "@highlightjs/vue-plugin";
hljs.registerLanguage('rust', rust);

1
frontend/src/index.template.html
View File

@ -13,6 +13,7 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/highlight.min.js"></script>
<!-- and it's easy to individually load additional languages -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/languages/rust.min.js"></script>
<!-- <script src="//cdn.jsdelivr.net/gh/TRSasasusu/highlightjs-highlight-lines.js@1.2.0/highlightjs-highlight-lines.min.js"></script> -->
<link rel="icon" type="image/ico" href="favicon.ico">

33
frontend/src/pages/IndexPage.vue
View File

@ -43,12 +43,15 @@
<q-item v-for="result in results" :key="result.title" @click="console.log(result)" class="">
<q-item-section>
<q-item-label>
{{ result.file }}:{{ result.start_line }} - &nbsp;&nbsp; <q-icon size="xs" name="open_in_news" />
<a :href="'https://github.com/qdrant/qdrant/blob/master/' + result.file + '#L' + (result.start_line + 1)"
{{ result.context.file_path }}:{{ result.line }} - &nbsp;&nbsp; <q-icon size="xs" name="open_in_news" />
<a :href="'https://github.com/qdrant/qdrant/blob/master/' + result.context.file_path + '#L' + result.line"
target="_blank">GitHub</a>
</q-item-label>
<q-item-label>
<highlightjs language="rust" :code="'...\n' + result.code_snippet.slice(0, 1000) + '\n...'" />
<!-- <pre v-highlightjs><code class="javascript">const s = new Date().toString()</code></pre> -->
<pre><code class="language-rust" v-html="result.context.snippet"></code></pre>
<!-- <highlightjs language="rust" :code="result.context.snippet" /> -->
</q-item-label>
</q-item-section>
@ -66,15 +69,19 @@ import { axios } from "boot/axios";
import { Notify } from 'quasar'
import hljs from 'highlight.js/lib/common';
import hljsVuePlugin from "@highlightjs/vue-plugin";
// Highlight lines plugin
// import hljsVuePlugin from "@highlightjs/vue-plugin";
// import { component as VueCodeHighlight } from 'vue-code-highlight';
let fakeData = [
{ "code_snippet": "fn estimate_should<F>(\n estimator: &F,\n conditions: &[Condition],\n total: usize,\n) -> CardinalityEstimation", "end_character": 1, "end_line": 127, "file": "lib/segment/src/index/query_estimator.rs", "start_character": 21, "start_line": 123 },
{ "code_snippet": " let query_cardinality = {\n let payload_index = self.payload_index.borrow();\n payload_index.estimate_cardinality(condition)\n };", "end_character": 17, "end_line": 701, "file": "lib/segment/src/segment.rs", "start_character": 40, "start_line": 698 },
{ "code_snippet": "fn estimate_must_not<F>(\n estimator: &F,\n conditions: &[Condition],\n total: usize,\n) -> CardinalityEstimation", "end_character": 1, "end_line": 162, "file": "lib/segment/src/index/query_estimator.rs", "start_character": 23, "start_line": 158 },
{ "code_snippet": " ) -> Option<CardinalityEstimation> {\n self.get_payload_field_index()\n .estimate_cardinality(condition)\n }", "end_character": 5, "end_line": 167, "file": "lib/segment/src/index/field_index/field_index_base.rs", "start_character": 39, "start_line": 164 },
{ "code_snippet": " fn cardinality_request(index: &NumericIndex<f64>, query: Range) -> CardinalityEstimation {\n let estimation = index.range_cardinality(&query);\n\n let result = index\n .filter(&FieldCondition::new_range(\"\".to_string(), query))\n .unwrap()\n .unique()\n .collect_vec();\n\n eprintln!(\"estimation = {:#?}\", estimation);\n eprintln!(\"result.len() = {:#?}\", result.len());\n assert!(estimation.min <= result.len());\n assert!(estimation.max >= result.len());\n estimation\n }", "end_character": 5, "end_line": 587, "file": "lib/segment/src/index/field_index/numeric_index.rs", "start_character": 93, "start_line": 573 },
{ "code_type": "Function", "context": { "file_name": "query_estimator.rs", "file_path": "lib/segment/src/index/query_estimator.rs", "module": "index", "snippet": "<mark>pub fn combine_should_estimations(</mark>\n<mark> estimations: &[CardinalityEstimation],</mark>\n<mark> total: usize,</mark>\n) -> CardinalityEstimation {\n let mut clauses: Vec<PrimaryCondition> = vec![];\n for estimation in estimations {\n if estimation.primary_clauses.is_empty() {\n // If some branch is un-indexed - we can't make\n // any assumptions about the whole `should` clause\n clauses = vec![];\n break;\n }\n clauses.append(&mut estimation.primary_clauses.clone());\n }\n let element_not_hit_prob: f64 = estimations\n .iter()\n .map(|x| (total - x.exp) as f64 / (total as f64))\n .product();\n let element_hit_prob = 1.0 - element_not_hit_prob;\n let expected_count = (element_hit_prob * (total as f64)).round() as usize;\n CardinalityEstimation {\n primary_clauses: clauses,\n min: estimations.iter().map(|x| x.min).max().unwrap_or(0),\n exp: expected_count,\n max: min(estimations.iter().map(|x| x.max).sum(), total),\n }\n}\n", "struct_name": null }, "docstring": null, "line": 13, "line_from": 13, "line_to": 39, "name": "combine_should_estimations", "signature": "fn combine_should_estimations (estimations : & [CardinalityEstimation] , total : usize ,) -> CardinalityEstimation", "sub_matches": [{ "overlap_from": 13, "overlap_to": 15 }] },
{ "code_type": "Function", "context": { "file_name": "query_estimator.rs", "file_path": "lib/segment/src/index/query_estimator.rs", "module": "index", "snippet": "<mark>fn estimate_must_not<F>(</mark>\n<mark> estimator: &F,</mark>\n<mark> conditions: &[Condition],</mark>\n<mark> total: usize,</mark>\n) -> CardinalityEstimation\nwhere\n F: Fn(&Condition) -> CardinalityEstimation,\n{\n let estimate = |x| invert_estimation(&estimate_condition(estimator, x, total), total);\n let must_not_estimations = conditions.iter().map(estimate).collect_vec();\n combine_must_estimations(&must_not_estimations, total)\n}\n", "struct_name": null }, "docstring": null, "line": 159, "line_from": 159, "line_to": 170, "name": "estimate_must_not", "signature": "fn estimate_must_not < F > (estimator : & F , conditions : & [Condition] , total : usize ,) -> CardinalityEstimation where F : Fn (& Condition) -> CardinalityEstimation ,", "sub_matches": [{ "overlap_from": 159, "overlap_to": 162 }] },
{ "code_type": "Function", "context": { "file_name": "query_estimator.rs", "file_path": "lib/segment/src/index/query_estimator.rs", "module": "index", "snippet": "<mark>fn estimate_condition<F>(</mark>\n<mark> estimator: &F,</mark>\n<mark> condition: &Condition,</mark>\n<mark> total: usize,</mark>\n) -> CardinalityEstimation\nwhere\n F: Fn(&Condition) -> CardinalityEstimation,\n{\n match condition {\n Condition::Filter(filter) => estimate_filter(estimator, filter, total),\n _ => estimator(condition),\n }\n}\n", "struct_name": null }, "docstring": null, "line": 76, "line_from": 76, "line_to": 88, "name": "estimate_condition", "signature": "fn estimate_condition < F > (estimator : & F , condition : & Condition , total : usize ,) -> CardinalityEstimation where F : Fn (& Condition) -> CardinalityEstimation ,", "sub_matches": [{ "overlap_from": 76, "overlap_to": 79 }] },
{ "code_type": "Function", "context": { "file_name": "query_estimator.rs", "file_path": "lib/segment/src/index/query_estimator.rs", "module": "index", "snippet": "pub fn combine_must_estimations(\n estimations: &[CardinalityEstimation],\n total: usize,\n) -> CardinalityEstimation {\n let min_estimation = estimations\n .iter()\n .map(|x| x.min)\n .fold(total as i64, |acc, x| {\n max(0, acc + (x as i64) - (total as i64))\n }) as usize;\n\n let max_estimation = estimations.iter().map(|x| x.max).min().unwrap_or(total);\n\n let exp_estimation_prob: f64 = estimations\n .iter()\n .map(|x| (x.exp as f64) / (total as f64))\n .product();\n\n let exp_estimation = (exp_estimation_prob * (total as f64)).round() as usize;\n\n let clauses = estimations\n .iter()\n .filter(|x| !x.primary_clauses.is_empty())\n .min_by_key(|x| x.exp)\n .map(|x| x.primary_clauses.clone())\n .unwrap_or_default();\n\n CardinalityEstimation {\n primary_clauses: clauses,\n min: min_estimation,\n exp: exp_estimation,\n max: max_estimation,\n }\n}\n", "struct_name": null }, "docstring": null, "line": 41, "line_from": 41, "line_to": 74, "name": "combine_must_estimations", "signature": "fn combine_must_estimations (estimations : & [CardinalityEstimation] , total : usize ,) -> CardinalityEstimation", "sub_matches": [] },
{ "code_type": "Function", "context": { "file_name": "query_estimator.rs", "file_path": "lib/segment/src/index/query_estimator.rs", "module": "index", "snippet": "fn estimate_must<F>(estimator: &F, conditions: &[Condition], total: usize) -> CardinalityEstimation\nwhere\n F: Fn(&Condition) -> CardinalityEstimation,\n{\n let estimate = |x| estimate_condition(estimator, x, total);\n let must_estimations = conditions.iter().map(estimate).collect_vec();\n\n combine_must_estimations(&must_estimations, total)\n}\n", "struct_name": null }, "docstring": null, "line": 137, "line_from": 137, "line_to": 145, "name": "estimate_must", "signature": "fn estimate_must < F > (estimator : & F , conditions : & [Condition] , total : usize) -> CardinalityEstimation where F : Fn (& Condition) -> CardinalityEstimation ,", "sub_matches": [] },
]
@ -82,20 +89,20 @@ export default defineComponent({
name: "IndexPage",
components: {
highlightjs: hljsVuePlugin.component
// highlightjs: hljsVuePlugin.component,
// VueCodeHighlight,
},
data: () => ({
query: "",
loading: false,
results: [],
results: fakeData,
showResults: true,
showQuickResults: false,
}),
created() {
// fetch on init
// this.search();
// hljs.highlightAll();
},
methods: {