use file path from env + reformat

This commit is contained in:
generall 2023-11-17 18:14:29 +01:00
parent 234b3f93fc
commit b817c98c8f
3 changed files with 7 additions and 11 deletions

View File

@ -5,9 +5,6 @@ from qdrant_client.http import models
from code_search.config import QDRANT_URL, QDRANT_API_KEY, QDRANT_FILE_COLLECTION_NAME
class FileGet:
def __init__(self):

View File

@ -5,10 +5,7 @@ import json
from code_search.config import QDRANT_URL, QDRANT_API_KEY, DATA_DIR, QDRANT_FILE_COLLECTION_NAME
def encode_and_upload():
qdrant_client = QdrantClient(
QDRANT_URL,
api_key=QDRANT_API_KEY,
@ -24,7 +21,6 @@ def encode_and_upload():
with open(input_file, 'r') as json_file:
data = json.load(json_file)
payload = data
print(f"Recreating the collection {collection_name}")
qdrant_client.recreate_collection(
@ -37,10 +33,9 @@ def encode_and_upload():
collection_name=collection_name,
payload=payload,
vectors=[{}] * len(payload),
ids=None,
ids=None,
batch_size=256
)
if __name__ == '__main__':

View File

@ -4,6 +4,7 @@ from pathlib import Path
from code_search.config import DATA_DIR
def process_file(root_dir, file_path):
with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:
code_lines = file.readlines()
@ -15,17 +16,19 @@ def process_file(root_dir, file_path):
"endline": len(code_lines)
}
def explore_directory(root_dir):
result = []
for foldername, subfolders, filenames in os.walk(root_dir):
for filename in filenames:
file_path = os.path.join(foldername, filename)
if file_path.endswith('.rs'):
if file_path.endswith('.rs'):
result.append(process_file(root_dir, file_path))
return result
def main():
folder_path = "/tmp/qdrant"
folder_path = os.getenv('QDRANT_PATH')
output_file = Path(DATA_DIR) / "rs_files.json"
files_data = explore_directory(folder_path)
@ -33,5 +36,6 @@ def main():
with open(output_file, 'w', encoding='utf-8') as json_file:
json.dump(files_data, json_file, indent=2)
if __name__ == "__main__":
main()