Add code search with zoekt support (#8827)

This PR adds zoekt as a code search engine for forgejo. This Pull Request is a continuation of the discussion #8302.
The meilisearch search engine was not suitable, as it is not designed for searching by code. The zoekt project was proposed instead. Zoekt copes well with code indexing, but its operating principle differs from such search engines as elasticsearch.
While elasticsearch can return a result in a ready-made form (with pagination, ready-made snippets, etc.) and forgejo only needs to show this result in the interface with a little work with the data, zoekt works completely differently.

Zoekt finds matches in the repository index and returns a response. The response contains a line with the search word, its number from the file, and also a context, if specified in the request. This response is not suitable for Forgejo, so you need to assemble it yourself. To assemble the response from Zoekt into a form acceptable for Forgejo, I had to write some code and create a new function `searchZoektResult`, since the existing `searchResult` function is completely unsuitable for this search engine. I also had to write logic for pagination, highlighting, and correct display of lines in found snippets with a match, but this is a feature of Zoekt.
At the moment, Zoekt does not support deleting a repository index by repo_id, it only supports complete deletion of all repositories. But I still implemented the Delete function, which deletes a specific repository by its ID.

Co-authored-by: Aleksandr Gamzin <gamzin@altlinux.org>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8827
Reviewed-by: Shiny Nematoda <snematoda@noreply.codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>

Author: 2 people

assets/go-licenses.json

@@ -1481,10 +1481,10 @@ LEVEL = Info
 ;; If empty then it defaults to `sources` only, as if you'd like to disable fully please see REPO_INDEXER_ENABLED.
 ;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates
 ;;
-;; Code search engine type, could be `bleve` or `elasticsearch`.
+;; Code search engine type, could be `bleve`, `zoekt` or `elasticsearch`.
 ;REPO_INDEXER_TYPE = bleve
 ;;
-;; Index file used for code search. available when `REPO_INDEXER_TYPE` is bleve
+;; Index file used for code search. available when `REPO_INDEXER_TYPE` is bleve or zoekt
 ;REPO_INDEXER_PATH = indexers/repos.bleve
 ;;
 ;; Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200
@@ -1494,10 +1494,10 @@ LEVEL = Info
 ;REPO_INDEXER_NAME = gitea_codes
 ;;
 ;; A comma separated list of glob patterns (see https://github.com/gobwas/glob) to include
-;; in the index; default is empty
+;; in the index; it's not compatible with the `zoekt` indexer type; default is empty
 ;REPO_INDEXER_INCLUDE =
 ;;
-;; A comma separated list of glob patterns to exclude from the index; ; default is empty
+;; A comma separated list of glob patterns to exclude from the index; it's not compatible with the `zoekt` indexer type; default is empty
 ;REPO_INDEXER_EXCLUDE =
 ;;
 ;; If vendored files should be excluded.

custom/conf/app.example.ini

@@ -92,6 +92,7 @@ require (
 	github.com/redis/go-redis/v9 v9.19.0
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
 	github.com/sergi/go-diff v1.4.0
+	github.com/sourcegraph/zoekt v0.0.0-20260114143800-c747a3bccc2a
 	github.com/stretchr/testify v1.11.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/ulikunitz/xz v0.5.15
@@ -117,10 +118,11 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	cloud.google.com/go/compute/metadata v0.7.0 // indirect
 	code.superseriousbusiness.org/go-png-image-structure/v2 v2.3.0 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
 	git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect
+	github.com/RoaringBitmap/roaring v1.9.4 // indirect
 	github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect
 	github.com/STARRY-S/zip v0.2.3 // indirect
 	github.com/andybalholm/brotli v1.2.0 // indirect
@@ -147,6 +149,7 @@ require (
 	github.com/blevesearch/zapx/v15 v15.4.3 // indirect
 	github.com/blevesearch/zapx/v16 v16.3.4 // indirect
 	github.com/blevesearch/zapx/v17 v17.1.2 // indirect
+	github.com/bmatcuk/doublestar v1.3.4 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/sevenzip v1.6.1 // indirect
@@ -195,6 +198,8 @@ require (
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
+	github.com/grafana/regexp v0.0.0-20240607082908-2cb410fa05da // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -226,6 +231,7 @@ require (
 	github.com/olekukonko/tablewriter v1.0.7 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.34.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -239,6 +245,7 @@ require (
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/sirupsen/logrus v1.9.4 // indirect
 	github.com/sorairolake/lzip-go v0.3.8 // indirect
+	github.com/sourcegraph/go-ctags v0.0.0-20250729094530-349a251d78d8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
@@ -256,6 +263,8 @@ require (
 	golang.org/x/mod v0.35.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
+	google.golang.org/grpc v1.75.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.mod

@@ -384,3 +384,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	}
 	return total, searchResults, searchResultLanguages, nil
 }
+
+func (b *Indexer) Formatter() internal.ResultFormatter {
+	return nil
+}

go.sum

@@ -418,3 +418,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 
 	return total, hits, extractAggs(countResult), err
 }
+
+func (b *Indexer) Formatter() internal.ResultFormatter {
+	return nil
+}

modules/indexer/code/bleve/bleve.go

@@ -0,0 +1,10 @@
+// Copyright 2026 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package code
+
+import "forgejo.org/modules/indexer/code/internal"
+
+func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges [][3]int, code string) []internal.ResultLine {
+	return internal.HighlightSearchResultCode(filename, lineNums, highlightRanges, code)
+}

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -17,6 +17,7 @@ import (
 	"forgejo.org/modules/indexer/code/bleve"
 	"forgejo.org/modules/indexer/code/elasticsearch"
 	"forgejo.org/modules/indexer/code/internal"
+	"forgejo.org/modules/indexer/code/zoekt"
 	"forgejo.org/modules/log"
 	"forgejo.org/modules/process"
 	"forgejo.org/modules/queue"
@@ -129,7 +130,7 @@ func Init() {
 
 	// Create the Queue
 	switch setting.Indexer.RepoType {
-	case "bleve", "elasticsearch":
+	case "bleve", "elasticsearch", "zoekt":
 		handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
 			indexer := *globalIndexer.Load()
 			// make it a process to allow for cancellation (especially during integration tests where no global shutdown happens)
@@ -200,6 +201,25 @@ func Init() {
 				close(waitChannel)
 				log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), util.SanitizeCredentialURLs(setting.Indexer.RepoConnStr), err)
 			}
+		case "zoekt":
+			log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoPath)
+			defer func() {
+				if err := recover(); err != nil {
+					log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2))
+					log.Error("The indexer files are likely corrupted and may need to be deleted")
+					log.Error("You can completely remove the \"%s\" directory to make Gitea recreate the indexes", setting.Indexer.RepoPath)
+				}
+			}()
+
+			rIndexer = zoekt.NewIndexer(setting.Indexer.RepoPath)
+			existed, err = rIndexer.Init(ctx)
+			if err != nil {
+				cancel()
+				(*globalIndexer.Load()).Close()
+				close(waitChannel)
+
+				log.Fatal("PID: %d Unable to initialize the zoekt Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err)
+			}
 
 		default:
 			log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType)

modules/indexer/code/highlight.go

@@ -0,0 +1,114 @@
+// Copyright 2026 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"bytes"
+	"html/template"
+	"strings"
+
+	"forgejo.org/modules/highlight"
+	"forgejo.org/services/gitdiff"
+)
+
+func WriteStrings(buf *bytes.Buffer, strs ...string) error {
+	for _, s := range strs {
+		_, err := buf.WriteString(s)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+const (
+	highlightTagStart = "<span class=\"search-highlight\">"
+	highlightTagEnd   = "</span>"
+)
+
+func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges [][3]int, code string) []ResultLine {
+	hcd := gitdiff.NewHighlightCodeDiff()
+	hcd.CollectUsedRunes(code)
+	startTag, endTag := hcd.NextPlaceholder(), hcd.NextPlaceholder()
+	hcd.PlaceholderTokenMap[startTag] = highlightTagStart
+	hcd.PlaceholderTokenMap[endTag] = highlightTagEnd
+
+	// we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting
+	hl, _ := highlight.Code(filename, "", code)
+	conv := hcd.ConvertToPlaceholders(string(hl))
+	convLines := strings.Split(conv, "\n")
+
+	// each highlightRange is of the form [line number, start byte offset, end byte offset]
+	for _, highlightRange := range highlightRanges {
+		ln, start, end := highlightRange[0], highlightRange[1], highlightRange[2]
+		line := convLines[ln]
+		if line == "" || len(line) <= start || len(line) < end {
+			continue
+		}
+
+		sr := strings.NewReader(line)
+		sb := strings.Builder{}
+		count := -1
+		isOpen := false
+		for r, size, err := sr.ReadRune(); err == nil; r, size, err = sr.ReadRune() {
+			if token, ok := hcd.PlaceholderTokenMap[r];
+			// token was not found
+			!ok {
+				count += size
+			} else if
+			// token was marked as used
+			token == "" ||
+				// the token is not an valid html tag emitted by chroma
+				!(len(token) > 6 && (token[0:5] == "<span" || token[0:6] == "</span")) {
+				count++
+			} else if !isOpen {
+				// open the tag only after all other placeholders
+				sb.WriteRune(r)
+				continue
+			} else if isOpen && count < end {
+				// if the tag is open, but a placeholder exists in between
+				// close the tag
+				sb.WriteRune(endTag)
+				// write the placeholder
+				sb.WriteRune(r)
+				// reopen the tag
+				sb.WriteRune(startTag)
+				continue
+			}
+
+			switch {
+			case count >= end:
+				// if tag is not open, no need to close
+				if !isOpen {
+					break
+				}
+				sb.WriteRune(endTag)
+				isOpen = false
+			case count >= start:
+				// if tag is open, do not open again
+				if isOpen {
+					break
+				}
+				isOpen = true
+				sb.WriteRune(startTag)
+			}
+
+			sb.WriteRune(r)
+		}
+		if isOpen {
+			sb.WriteRune(endTag)
+		}
+		convLines[ln] = sb.String()
+	}
+	conv = strings.Join(convLines, "\n")
+
+	highlightedLines := strings.Split(hcd.Recover(conv), "\n")
+	// The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n`
+	lines := make([]ResultLine, min(len(highlightedLines), len(lineNums)))
+	for i := range len(lines) {
+		lines[i].Num = lineNums[i]
+		lines[i].FormattedContent = template.HTML(highlightedLines[i])
+	}
+	return lines
+}

modules/indexer/code/indexer.go

@@ -18,6 +18,7 @@ type Indexer interface {
 	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
 	Delete(ctx context.Context, repoID int64) error
 	Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error)
+	Formatter() ResultFormatter
 }
 
 type CodeSearchMode int
@@ -72,3 +73,7 @@ func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error {
 func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	return 0, nil, nil, errors.New("indexer is not ready")
 }
+
+func (d *dummyIndexer) Formatter() ResultFormatter {
+	return nil
+}