133728e barerepo 1mo
1
package search
133728e barerepo 1mo
3
import (
133728e barerepo 1mo
4
"context"
133728e barerepo 1mo
5
"strconv"
133728e barerepo 1mo
6
"strings"
133728e barerepo 1mo
8
"github.com/barerepo/server/internal/gitx"
133728e barerepo 1mo
9
"github.com/barerepo/server/internal/repocfg"
133728e barerepo 1mo
10
"github.com/barerepo/server/internal/store"
133728e barerepo 1mo
11
"github.com/barerepo/server/internal/thread"
133728e barerepo 1mo
14
// maxIndexedBytes keeps one generated file out of the whole index. Chapter 17 indexes source.
133728e barerepo 1mo
15
const maxIndexedBytes = 512 << 10
133728e barerepo 1mo
17
// Index is what the indexer needs from the database, an interface because a hook is its own process.
133728e barerepo 1mo
18
type Index interface {
133728e barerepo 1mo
19
PutDoc(ctx context.Context, d store.Doc, public bool, readers string) error
133728e barerepo 1mo
20
PutDocs(ctx context.Context, repo, kind string, docs []store.Doc, public bool, readers string) error
133728e barerepo 1mo
21
DeleteDoc(ctx context.Context, repo, kind, path string) error
133728e barerepo 1mo
22
SetReadable(ctx context.Context, repo string, public bool, readers string) error
133728e barerepo 1mo
23
HasDocs(ctx context.Context, repo string) (bool, error)
133728e barerepo 1mo
26
// Target names one repository to the indexer, with everything the read filter needs.
133728e barerepo 1mo
27
type Target struct {
133728e barerepo 1mo
28
Owner string
133728e barerepo 1mo
29
Name string
133728e barerepo 1mo
30
Dir string
133728e barerepo 1mo
31
Ref string
133728e barerepo 1mo
32
Config repocfg.Config
133728e barerepo 1mo
35
func (r Target) full() string { return r.Owner + "/" + r.Name }
133728e barerepo 1mo
37
func (r Target) readers() string { return store.Readers(r.Owner, r.Config.Access.Push) }
133728e barerepo 1mo
39
// IndexAll rebuilds every document for one repository, for a first push and for a reindex.
133728e barerepo 1mo
40
func IndexAll(ctx context.Context, db Index, r Target) error {
133728e barerepo 1mo
41
if err := indexCodeAll(ctx, db, r); err != nil {
133728e barerepo 1mo
42
return err
133728e barerepo 1mo
44
if err := IndexThreads(ctx, db, r); err != nil {
133728e barerepo 1mo
45
return err
133728e barerepo 1mo
47
return indexRepoRow(ctx, db, r)
133728e barerepo 1mo
50
// IndexMeta rewrites what any push can change without touching a file: the read set and the talk.
133728e barerepo 1mo
51
func IndexMeta(ctx context.Context, db Index, r Target) error {
133728e barerepo 1mo
52
if err := db.SetReadable(ctx, r.full(), r.Config.Public(), r.readers()); err != nil {
133728e barerepo 1mo
53
return err
133728e barerepo 1mo
55
if err := indexRepoRow(ctx, db, r); err != nil {
133728e barerepo 1mo
56
return err
133728e barerepo 1mo
58
return IndexThreads(ctx, db, r)
133728e barerepo 1mo
61
// IndexPush walks only the paths the push changed, which is what chapter 17 means by incremental.
133728e barerepo 1mo
62
func IndexPush(ctx context.Context, db Index, r Target, old, new string) error {
133728e barerepo 1mo
63
indexed, err := db.HasDocs(ctx, r.full())
133728e barerepo 1mo
64
if err != nil {
133728e barerepo 1mo
65
return err
133728e barerepo 1mo
67
if !indexed || !gitx.ValidRev(old) || strings.Trim(old, "0") == "" {
133728e barerepo 1mo
68
return indexCodeAll(ctx, db, r)
133728e barerepo 1mo
70
changed, err := gitx.Run(ctx, r.Dir, "diff", "--name-only", "--no-renames", old, new)
133728e barerepo 1mo
71
if err != nil {
133728e barerepo 1mo
72
return indexCodeAll(ctx, db, r)
133728e barerepo 1mo
74
var paths []string
133728e barerepo 1mo
75
for _, p := range strings.Split(strings.TrimRight(changed, "\n"), "\n") {
133728e barerepo 1mo
76
if p != "" {
133728e barerepo 1mo
77
paths = append(paths, p)
133728e barerepo 1mo
80
return indexCodePaths(ctx, db, r, paths)
133728e barerepo 1mo
83
// indexCodeAll reads every path at the tip, which is the one place a whole tree is walked.
133728e barerepo 1mo
84
func indexCodeAll(ctx context.Context, db Index, r Target) error {
133728e barerepo 1mo
85
if !gitx.ValidRev(r.Ref) {
133728e barerepo 1mo
86
return nil
133728e barerepo 1mo
88
out, err := gitx.Run(ctx, r.Dir, "ls-tree", "-r", "--name-only", r.Ref)
133728e barerepo 1mo
89
if err != nil {
133728e barerepo 1mo
90
return nil
133728e barerepo 1mo
92
var paths []string
133728e barerepo 1mo
93
for _, p := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
133728e barerepo 1mo
94
if p != "" {
133728e barerepo 1mo
95
paths = append(paths, p)
133728e barerepo 1mo
98
docs, err := readBlobs(ctx, r, paths)
133728e barerepo 1mo
99
if err != nil {
133728e barerepo 1mo
100
return err
133728e barerepo 1mo
101
}
133728e barerepo 1mo
102
return db.PutDocs(ctx, r.full(), store.Code, docs, r.Config.Public(), r.readers())
133728e barerepo 1mo
105
// indexCodePaths updates the paths a push touched, and drops the ones it removed.
133728e barerepo 1mo
106
func indexCodePaths(ctx context.Context, db Index, r Target, paths []string) error {
133728e barerepo 1mo
107
docs, err := readBlobs(ctx, r, paths)
133728e barerepo 1mo
108
if err != nil {
133728e barerepo 1mo
109
return err
133728e barerepo 1mo
110
}
133728e barerepo 1mo
111
kept := make(map[string]bool, len(docs))
133728e barerepo 1mo
112
for _, d := range docs {
133728e barerepo 1mo
113
kept[d.Path] = true
133728e barerepo 1mo
114
if err := db.PutDoc(ctx, d, r.Config.Public(), r.readers()); err != nil {
133728e barerepo 1mo
115
return err
133728e barerepo 1mo
116
}
133728e barerepo 1mo
117
}
133728e barerepo 1mo
118
for _, p := range paths {
133728e barerepo 1mo
119
if kept[p] {
133728e barerepo 1mo
120
continue
133728e barerepo 1mo
121
}
133728e barerepo 1mo
122
if err := db.DeleteDoc(ctx, r.full(), store.Code, p); err != nil {
133728e barerepo 1mo
123
return err
133728e barerepo 1mo
124
}
133728e barerepo 1mo
125
}
133728e barerepo 1mo
126
return nil
133728e barerepo 1mo
129
// readBlobs reads the named paths at the tip through the object pool, skipping what is not source.
133728e barerepo 1mo
130
func readBlobs(ctx context.Context, r Target, paths []string) ([]store.Doc, error) {
133728e barerepo 1mo
131
if len(paths) == 0 || !gitx.ValidRev(r.Ref) {
133728e barerepo 1mo
132
return nil, nil
133728e barerepo 1mo
133
}
133728e barerepo 1mo
134
specs := make([]string, 0, len(paths))
133728e barerepo 1mo
135
for _, p := range paths {
133728e barerepo 1mo
136
specs = append(specs, r.Ref+":"+p)
133728e barerepo 1mo
137
}
133728e barerepo 1mo
138
objs, err := gitx.Batch(ctx, r.Dir, specs)
133728e barerepo 1mo
139
if err != nil {
133728e barerepo 1mo
140
return nil, err
133728e barerepo 1mo
141
}
133728e barerepo 1mo
142
docs := make([]store.Doc, 0, len(paths))
133728e barerepo 1mo
143
for i, p := range paths {
133728e barerepo 1mo
144
obj := objs[specs[i]]
133728e barerepo 1mo
145
if obj == nil || obj.Type != "blob" || obj.Size > maxIndexedBytes {
133728e barerepo 1mo
146
continue
133728e barerepo 1mo
147
}
133728e barerepo 1mo
148
if strings.IndexByte(obj.Body, 0) >= 0 {
133728e barerepo 1mo
149
continue
133728e barerepo 1mo
150
}
133728e barerepo 1mo
151
docs = append(docs, store.Doc{Repo: r.full(), Kind: store.Code, Path: p, Body: obj.Body})
133728e barerepo 1mo
152
}
133728e barerepo 1mo
153
return docs, nil
133728e barerepo 1mo
156
// IndexThreads rewrites the discussion, which chapter 17 asks for on note write and not on query.
133728e barerepo 1mo
157
func IndexThreads(ctx context.Context, db Index, r Target) error {
133728e barerepo 1mo
158
list, err := thread.List(ctx, r.Dir)
133728e barerepo 1mo
159
if err != nil {
133728e barerepo 1mo
160
return nil
133728e barerepo 1mo
161
}
133728e barerepo 1mo
162
docs := make([]store.Doc, 0, len(list))
133728e barerepo 1mo
163
for _, sum := range list {
133728e barerepo 1mo
164
_, comments, err := thread.Read(ctx, r.Dir, sum.N)
133728e barerepo 1mo
165
if err != nil {
133728e barerepo 1mo
166
continue
133728e barerepo 1mo
167
}
133728e barerepo 1mo
168
var body strings.Builder
133728e barerepo 1mo
169
for _, c := range comments {
133728e barerepo 1mo
170
body.WriteString(c.Body)
133728e barerepo 1mo
171
body.WriteByte('\n')
133728e barerepo 1mo
172
}
133728e barerepo 1mo
173
docs = append(docs, store.Doc{Repo: r.full(), Kind: store.Thread,
133728e barerepo 1mo
174
Path: strconv.Itoa(sum.N), Title: sum.Meta.Title, Body: body.String()})
133728e barerepo 1mo
175
}
133728e barerepo 1mo
176
return db.PutDocs(ctx, r.full(), store.Thread, docs, r.Config.Public(), r.readers())
133728e barerepo 1mo
179
// indexRepoRow is the repository itself, matched on its name and its description.
133728e barerepo 1mo
180
func indexRepoRow(ctx context.Context, db Index, r Target) error {
133728e barerepo 1mo
181
doc := store.Doc{Repo: r.full(), Kind: store.Repository,
133728e barerepo 1mo
182
Title: r.full(), Body: r.Config.Repo.Description}
133728e barerepo 1mo
183
return db.PutDoc(ctx, doc, r.Config.Public(), r.readers())