dnrops
/
datastar
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix in memory fuzzy search embedded pathing

This commit is contained in:
Delaney Gillilan 2025-02-08 19:11:01 -08:00
parent 860aa9e46d
commit 4042dc0e25
1 changed files with 16 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
site/router.go
View File

@ -8,8 +8,8 @@ import (
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
"time"
"github.com/a-h/templ"
@ -17,6 +17,7 @@ import (
"github.com/blevesearch/bleve/v2"
"github.com/delaneyj/toolbelt"
"github.com/delaneyj/toolbelt/embeddednats"
"github.com/dustin/go-humanize"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/gorilla/sessions"
@ -30,7 +31,6 @@ var staticFS embed.FS
var (
staticSys = hashfs.NewFS(staticFS)
highlightCSS templ.Component
indexPath = filepath.Join(os.TempDir(), "data-star.bleve")
)
func staticPath(path string) string {
@ -106,26 +106,21 @@ func setupRoutes(ctx context.Context, router chi.Router) (err error) {
}
ns.WaitForServer()
index, err := bleve.Open(indexPath)
if err == bleve.ErrorIndexPathDoesNotExist {
log.Printf("Creating new index...")
mapping := bleve.NewIndexMapping()
index, err = bleve.New(indexPath, mapping)
if err != nil {
log.Fatal(fmt.Errorf("failed to create index: %w", err))
}
} else if err != nil {
log.Fatal("failed to open index: %w", err)
} else {
log.Printf("Opening existing index...")
log.Printf("Creating new in memory index...")
memStats := &runtime.MemStats{}
runtime.ReadMemStats(memStats)
before := memStats.HeapInuse
mapping := bleve.NewIndexMapping()
index, err := bleve.NewMemOnly(mapping)
if err != nil {
log.Fatal(fmt.Errorf("failed to create index: %w", err))
}
if err := indexSiteContent(ctx, index); err != nil {
log.Fatal("failed to index site content, ", err)
}
log.Println("Indexed site, index can be found at: ", indexPath)
runtime.ReadMemStats(memStats)
after := memStats.HeapInuse
log.Printf("Indexing took %s", humanize.Bytes(after-before))
sessionSignals := sessions.NewCookieStore([]byte("datastar-session-secret"))
sessionSignals.MaxAge(int(24 * time.Hour / time.Second))
@ -154,10 +149,10 @@ type SiteIndexDoc struct {
}
func indexSiteContent(ctx context.Context, index bleve.Index) error {
markdownDir := "./site/static/md"
markdownDir := "static/md"
extractor := plaintext.NewHtmlExtractor()
return filepath.WalkDir(markdownDir, func(path string, d fs.DirEntry, err error) error {
return fs.WalkDir(staticFS, markdownDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return fmt.Errorf("error accessing path %s: %w", path, err)
}
@ -168,8 +163,7 @@ func indexSiteContent(ctx context.Context, index bleve.Index) error {
return fmt.Errorf("failed to compute relative path for %s: %w", path, err)
}
log.Printf("Indexing directory: %s", relDirName)
// log.Printf("Indexing directory: %s", relDirName)
dataset, err := markdownRenders(ctx, relDirName)
if err != nil {
return fmt.Errorf("failed to render markdown directory %s: %w", relDirName, err)