use file path from env + reformat
This commit is contained in:
parent
234b3f93fc
commit
b817c98c8f
|
@ -5,9 +5,6 @@ from qdrant_client.http import models
|
|||
|
||||
from code_search.config import QDRANT_URL, QDRANT_API_KEY, QDRANT_FILE_COLLECTION_NAME
|
||||
|
||||
|
||||
|
||||
|
||||
class FileGet:
|
||||
|
||||
def __init__(self):
|
||||
|
|
|
@ -5,10 +5,7 @@ import json
|
|||
from code_search.config import QDRANT_URL, QDRANT_API_KEY, DATA_DIR, QDRANT_FILE_COLLECTION_NAME
|
||||
|
||||
|
||||
|
||||
|
||||
def encode_and_upload():
|
||||
|
||||
qdrant_client = QdrantClient(
|
||||
QDRANT_URL,
|
||||
api_key=QDRANT_API_KEY,
|
||||
|
@ -24,7 +21,6 @@ def encode_and_upload():
|
|||
with open(input_file, 'r') as json_file:
|
||||
data = json.load(json_file)
|
||||
payload = data
|
||||
|
||||
|
||||
print(f"Recreating the collection {collection_name}")
|
||||
qdrant_client.recreate_collection(
|
||||
|
@ -37,10 +33,9 @@ def encode_and_upload():
|
|||
collection_name=collection_name,
|
||||
payload=payload,
|
||||
vectors=[{}] * len(payload),
|
||||
ids=None,
|
||||
ids=None,
|
||||
batch_size=256
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
@ -4,6 +4,7 @@ from pathlib import Path
|
|||
|
||||
from code_search.config import DATA_DIR
|
||||
|
||||
|
||||
def process_file(root_dir, file_path):
|
||||
with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:
|
||||
code_lines = file.readlines()
|
||||
|
@ -15,17 +16,19 @@ def process_file(root_dir, file_path):
|
|||
"endline": len(code_lines)
|
||||
}
|
||||
|
||||
|
||||
def explore_directory(root_dir):
|
||||
result = []
|
||||
for foldername, subfolders, filenames in os.walk(root_dir):
|
||||
for filename in filenames:
|
||||
file_path = os.path.join(foldername, filename)
|
||||
if file_path.endswith('.rs'):
|
||||
if file_path.endswith('.rs'):
|
||||
result.append(process_file(root_dir, file_path))
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
folder_path = "/tmp/qdrant"
|
||||
folder_path = os.getenv('QDRANT_PATH')
|
||||
output_file = Path(DATA_DIR) / "rs_files.json"
|
||||
|
||||
files_data = explore_directory(folder_path)
|
||||
|
@ -33,5 +36,6 @@ def main():
|
|||
with open(output_file, 'w', encoding='utf-8') as json_file:
|
||||
json.dump(files_data, json_file, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
Loading…
Reference in New Issue