Skip to content

Commit fe2ed5c

Browse files
chrismwendtchrismwendt
authored
Fix build: revert sqlite changes (sourcegraph#2414)
1 parent 64aad09 commit fe2ed5c

25 files changed

Lines changed: 267 additions & 834 deletions

‎.gitignore‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,3 @@ cmd/management-console/assets/favicon.*.png
125125
cmd/management-console/web/src/favicon.png
126126
cmd/management-console/web/.cache
127127
cmd/management-console/assets/assets_vfsdata.go
128-
129-
# Symbols service: PCRE extension to sqlite3
130-
libsqlite3-pcre.dylib
131-
libsqlite3-pcre.so

‎CHANGELOG.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ All notable changes to Sourcegraph are documented in this file.
1111

1212
### Changed
1313

14-
- Symbols search is much faster now. After the initial indexing, you can expect code intelligence to be nearly instant no matter the size of your repository.
15-
1614
### Fixed
1715

1816
### Removed

‎cmd/symbols/Dockerfile‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,15 @@
1-
FROM alpine:3.9 AS libsqlite3-pcre
1+
FROM alpine:3.9 AS ctags
22

33
# hadolint ignore=DL3003,DL3018,DL4006
4-
RUN apk --no-cache add --virtual build-deps curl git gcc make libc-dev pcre-dev sqlite-dev && \
5-
mkdir /sqlite3-pcre && \
6-
curl -fsSL https://codeload.github.com/ralight/sqlite3-pcre/tar.gz/c98da412b431edb4db22d3245c99e6c198d49f7a | tar -C /sqlite3-pcre -xzvf - --strip 1 && \
7-
cd /sqlite3-pcre && \
8-
make && \
9-
apk --no-cache --purge del build-deps
4+
RUN apk --no-cache add --virtual build-deps curl jansson-dev libseccomp-dev linux-headers autoconf pkgconfig make automake gcc g++ binutils && curl https://codeload.github.com/universal-ctags/ctags/tar.gz/08a69b5a030cf56c079e3006238b992b34c9cc51 | tar xz -C /tmp && cd /tmp/ctags-08a69b5a030cf56c079e3006238b992b34c9cc51 && ./autogen.sh && LDFLAGS=-static ./configure --program-prefix=universal- --enable-json --enable-seccomp && make -j8 && make install && cd && rm -rf /tmp/ctags-08a69b5a030cf56c079e3006238b992b34c9cc51 && apk --no-cache --purge del build-deps
105

116
FROM alpine:3.9
127

138
# hadolint ignore=DL3018
149
RUN apk add --no-cache bind-tools ca-certificates mailcap tini
1510

16-
# hadolint ignore=DL3022
1711
COPY --from=ctags /usr/local/bin/universal-* /usr/local/bin/
1812

19-
COPY --from=libsqlite3-pcre /sqlite3-pcre/pcre.so /libsqlite3-pcre.so
20-
ENV LIBSQLITE3_PCRE /libsqlite3-pcre.so
21-
# hadolint ignore=DL3018
22-
RUN apk --no-cache add pcre-dev
23-
2413
# universal-ctags doesn't support some popular languages out of the box (Scala,
2514
# Swift, and Haskell, to name a few). However, you can add language support by
2615
# providing regexes on the command line or in a configuration file.

‎cmd/symbols/build.sh‎

Lines changed: 23 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,162 +1,33 @@
11
#!/usr/bin/env bash
22

3+
# We want to build multiple go binaries, so we use a custom build step on CI.
4+
cd $(dirname "${BASH_SOURCE[0]}")/../..
35
set -e
46

5-
SYMBOLS_IMAGE="${SYMBOLS_IMAGE:=dev-symbols}"
6-
CTAGS_IMAGE="${CTAGS_IMAGE:=ctags}"
7-
BUILD_TYPE="${BUILD_TYPE:=dev}"
8-
9-
repositoryRoot="$PWD"
10-
symbolsExecutablePath="$repositoryRoot/.bin/symbols"
11-
case "$OSTYPE" in
12-
darwin*)
13-
libsqlite3PcrePath="$repositoryRoot/libsqlite3-pcre.dylib"
14-
;;
15-
linux*)
16-
libsqlite3PcrePath="$repositoryRoot/libsqlite3-pcre.so"
17-
;;
18-
*)
19-
echo "Unknown platform $OSTYPE"
20-
exit 1
21-
;;
22-
esac
23-
24-
# Builds the PCRE extension to sqlite3.
25-
function buildLibsqlite3Pcre() {
26-
if ! command -v pkg-config >/dev/null 2>&1 || ! command -v pkg-config --cflags sqlite3 libpcre >/dev/null 2>&1; then
27-
echo "Missing sqlite dependencies."
28-
case "$OSTYPE" in
29-
darwin*)
30-
echo "Install them by running `brew install pkg-config sqlite pcre FiloSottile/musl-cross/musl-cross`"
31-
;;
32-
linux*)
33-
echo "Install them by running `apt-get install libpcre3-dev libsqlite3-dev pkg-config musl-tools`"
34-
;;
35-
*)
36-
echo "See the local development documentation: https://github.com/sourcegraph/sourcegraph/blob/master/doc/dev/local_development.md#step-2-install-dependencies"
37-
;;
38-
esac
39-
exit 1
40-
fi
41-
42-
43-
if [ -f "$libsqlite3PcrePath" ]; then
44-
return
45-
fi
46-
47-
sqlite3PcreRepositoryDirectory="$(mktemp -d)"
48-
trap "rm -rf $sqlite3PcreRepositoryDirectory" EXIT
49-
50-
echo "Building $libsqlite3PcrePath..."
51-
curl -fsSL https://codeload.github.com/ralight/sqlite3-pcre/tar.gz/c98da412b431edb4db22d3245c99e6c198d49f7a | tar -C "$sqlite3PcreRepositoryDirectory" -xzvf - --strip 1
52-
pushd "$sqlite3PcreRepositoryDirectory"
53-
case "$OSTYPE" in
54-
darwin*)
55-
# pkg-config spits out multiple arguments and must not be quoted.
56-
gcc -fno-common -dynamiclib pcre.c -o "$libsqlite3PcrePath" $(pkg-config --cflags sqlite3 libpcre) $(pkg-config --libs libpcre) -fPIC
57-
;;
58-
linux*)
59-
# pkg-config spits out multiple arguments and must not be quoted.
60-
gcc -shared -o "$libsqlite3PcrePath" $(pkg-config --cflags sqlite3 libpcre) -fPIC -W -Werror pcre.c $(pkg-config --libs libpcre) -Wl,-z,defs
61-
;;
62-
*)
63-
echo "Unknown platform $OSTYPE"
64-
exit 1
65-
;;
66-
esac
67-
popd
68-
echo "Building $libsqlite3PcrePath... done"
7+
OUTPUT=`mktemp -d -t sgdockerbuild_XXXXXXX`
8+
cleanup() {
9+
rm -rf "$OUTPUT"
6910
}
11+
trap cleanup EXIT
7012

71-
# Builds the symbols executable.
72-
function buildExecutable() {
73-
symbolsPackage="github.com/sourcegraph/sourcegraph/cmd/symbols"
74-
75-
case "$BUILD_TYPE" in
76-
dev)
77-
gcFlags="all=-N -l"
78-
tags="dev delve"
79-
;;
80-
dist)
81-
gcFlags=""
82-
tags="dist"
83-
;;
84-
esac
85-
86-
echo "Building the $symbolsExecutablePath executable..."
87-
go build -buildmode exe -gcflags "$gcFlags" -tags "$tags" -o "$symbolsExecutablePath" "$symbolsPackage"
88-
echo "Building the $symbolsExecutablePath executable... done"
89-
}
90-
91-
# Builds and runs the symbols executable.
92-
function execute() {
93-
buildLibsqlite3Pcre
94-
buildExecutable
95-
buildCtagsDockerImage
96-
export LIBSQLITE3_PCRE="$libsqlite3PcrePath"
97-
export CTAGS_COMMAND="${CTAGS_COMMAND:=cmd/symbols/universal-ctags-dev}"
98-
export CTAGS_PROCESSES="${CTAGS_PROCESSES:=1}"
99-
"$symbolsExecutablePath"
100-
}
13+
# Environment for building linux binaries
14+
export GO111MODULE=on
15+
export GOARCH=amd64
16+
export GOOS=linux
17+
export CGO_ENABLED=0
10118

102-
# Builds the symbols Docker image.
103-
function buildSymbolsDockerImage() {
104-
case "$OSTYPE" in
105-
darwin*)
106-
muslGcc="x86_64-linux-musl-gcc"
107-
if ! command -v "$muslGcc" >/dev/null 2>&1; then
108-
echo "Couldn't find musl C compiler $muslGcc. Run `brew install FiloSottile/musl-cross/musl-cross`."
109-
exit 1
110-
fi
111-
;;
112-
linux*)
113-
muslGcc="musl-gcc"
114-
if ! command -v "$muslGcc" >/dev/null 2>&1; then
115-
echo "Couldn't find musl C compiler $muslGcc. Install the musl-tools package (e.g. on Ubuntu, run `apt-get install musl-tools`)."
116-
exit 1
117-
fi
118-
;;
119-
*)
120-
echo "Unknown platform $OSTYPE"
121-
exit 1
122-
;;
123-
esac
19+
echo "Compiling the symbols service..."
20+
for pkg in github.com/sourcegraph/sourcegraph/cmd/symbols; do
21+
go build -ldflags "-X github.com/sourcegraph/sourcegraph/pkg/version.version=$VERSION" -buildmode exe -tags dist -o $OUTPUT/$(basename $pkg) $pkg
22+
done
12423

125-
symbolsDockerBuildContext="$(mktemp -d)"
126-
trap "rm -rf $symbolsDockerBuildContext" EXIT
24+
mkdir "$OUTPUT/.ctags.d"
25+
cp cmd/symbols/.ctags.d/additional-languages.ctags "$OUTPUT/.ctags.d/additional-languages.ctags"
12726

128-
export CC="$muslGcc"
129-
export GO111MODULE=on
130-
export GOARCH=amd64
131-
export GOOS=linux
132-
export CGO_ENABLED=1 # to build the sqlite3 library
133-
symbolsExecutablePath="$symbolsDockerBuildContext/symbols"
134-
buildExecutable
135-
136-
buildCtagsDockerImage
137-
138-
cp -R cmd/symbols/.ctags.d "$symbolsDockerBuildContext"
139-
140-
echo "Building the $SYMBOLS_IMAGE Docker image..."
141-
docker build --quiet -f cmd/symbols/Dockerfile -t "$SYMBOLS_IMAGE" "$symbolsDockerBuildContext"
142-
echo "Building the $SYMBOLS_IMAGE Docker image... done"
143-
}
144-
145-
# Builds the ctags docker image, used by universal-ctags-dev and the symbols Docker image.
146-
function buildCtagsDockerImage() {
147-
ctagsDockerBuildContext="$(mktemp -d)"
148-
trap "rm -rf $ctagsDockerBuildContext" EXIT
149-
150-
cp -R cmd/symbols/.ctags.d "$ctagsDockerBuildContext"
151-
152-
echo "Building the $CTAGS_IMAGE Docker image..."
153-
docker build --quiet -f cmd/symbols/internal/pkg/ctags/Dockerfile -t "$CTAGS_IMAGE" "$ctagsDockerBuildContext"
154-
echo "Building the $CTAGS_IMAGE Docker image... done"
155-
}
156-
157-
command="$1"
158-
if type -t "$command" >/dev/null 2>&1; then
159-
"$command"
160-
else
161-
echo "Unknown command: $command (must be: execute, buildLibsqlite3Pcre, or buildSymbolsDockerImage)"
27+
if [ -z "$IMAGE" ]; then
28+
echo "You have to set \$IMAGE, the tag for the symbols image."
29+
exit 1
16230
fi
31+
32+
echo "Building symbols image $IMAGE..."
33+
docker build --quiet -f cmd/symbols/Dockerfile -t $IMAGE $OUTPUT

‎cmd/symbols/internal/pkg/ctags/Dockerfile‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎cmd/symbols/internal/pkg/ctags/parser.go‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ import (
1010
"log"
1111
"os"
1212
"os/exec"
13-
14-
"github.com/sourcegraph/sourcegraph/pkg/env"
1513
)
1614

1715
type Entry struct {
@@ -35,26 +33,6 @@ type Parser interface {
3533
Close()
3634
}
3735

38-
func isCommandAvailable(name string) bool {
39-
cmd := exec.Command("/bin/sh", "-c", "command -v "+name)
40-
if err := cmd.Run(); err != nil {
41-
return false
42-
}
43-
return true
44-
}
45-
46-
var ctagsCommand = env.Get("CTAGS_COMMAND", "universal-ctags", "ctags command (should point to universal-ctags executable compiled with JSON and seccomp support)")
47-
48-
// GetCommand returns the ctags command from the CTAGS_COMMAND environment
49-
// variable, falling back to `universal-ctags`. Panics if the command doesn't
50-
// exist.
51-
func GetCommand() string {
52-
if !isCommandAvailable(ctagsCommand) {
53-
panic(fmt.Errorf("ctags command %s not found", ctagsCommand))
54-
}
55-
return ctagsCommand
56-
}
57-
5836
func NewParser(ctagsCommand string) (Parser, error) {
5937
opt := "default"
6038

‎cmd/symbols/internal/symbols/index.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package symbols
2+
3+
import (
4+
"bytes"
5+
"compress/gzip"
6+
"context"
7+
"encoding/gob"
8+
"io"
9+
"io/ioutil"
10+
11+
opentracing "github.com/opentracing/opentracing-go"
12+
"github.com/opentracing/opentracing-go/ext"
13+
otlog "github.com/opentracing/opentracing-go/log"
14+
"github.com/sourcegraph/sourcegraph/pkg/api"
15+
"github.com/sourcegraph/sourcegraph/pkg/symbols/protocol"
16+
"golang.org/x/net/trace"
17+
)
18+
19+
func (s *Service) indexedSymbols(ctx context.Context, repo api.RepoName, commitID api.CommitID) (symbols []protocol.Symbol, err error) {
20+
span, ctx := opentracing.StartSpanFromContext(ctx, "indexedSymbols")
21+
defer func() {
22+
if err != nil {
23+
ext.Error.Set(span, true)
24+
span.LogFields(otlog.Error(err))
25+
}
26+
span.Finish()
27+
}()
28+
29+
key := string(repo) + ":" + string(commitID) + ":v1" // suffix is index format version (vN)
30+
31+
tr := trace.New("indexedSymbols", string(repo))
32+
tr.LazyPrintf("commitID: %s", commitID)
33+
34+
var fetched bool
35+
defer func() {
36+
tr.LazyPrintf("fetched=%v symbols=%d", fetched, len(symbols))
37+
if err != nil {
38+
tr.LazyPrintf("error: %s", err)
39+
tr.SetError()
40+
}
41+
tr.Finish()
42+
}()
43+
44+
f, err := s.cache.Open(ctx, key, func(ctx context.Context) (io.ReadCloser, error) {
45+
fetched = true
46+
47+
var err error
48+
symbols, err = s.parseUncached(ctx, repo, commitID)
49+
if err != nil {
50+
return nil, err
51+
}
52+
return encodeSymbols(symbols)
53+
})
54+
if err != nil {
55+
return nil, err
56+
}
57+
defer f.Close()
58+
59+
// Skip deserializing symbols if we just serialized them in the s.cache.Open call above and still have the
60+
// slice.
61+
if symbols == nil {
62+
var size int64
63+
if fi, err := f.Stat(); err == nil {
64+
size = fi.Size()
65+
}
66+
tr.LazyPrintf("decode bytes=%d", size)
67+
symbols, err = decodeSymbols(ctx, f)
68+
tr.LazyPrintf("decode (done) symbols=%d", len(symbols))
69+
if err != nil {
70+
return nil, err
71+
}
72+
}
73+
74+
span.LogFields(otlog.String("event", "result"), otlog.Int("count", len(symbols)))
75+
return symbols, nil
76+
}
77+
78+
func encodeSymbols(symbols []protocol.Symbol) (io.ReadCloser, error) {
79+
var buf bytes.Buffer
80+
zw := gzip.NewWriter(&buf)
81+
enc := gob.NewEncoder(zw)
82+
if err := enc.Encode(symbols); err != nil {
83+
return nil, err
84+
}
85+
if err := zw.Close(); err != nil {
86+
return nil, err
87+
}
88+
return ioutil.NopCloser(&buf), nil
89+
}
90+
91+
func decodeSymbols(ctx context.Context, r io.Reader) (symbols []protocol.Symbol, err error) {
92+
span, _ := opentracing.StartSpanFromContext(ctx, "decodeSymbols")
93+
defer func() {
94+
if err != nil {
95+
ext.Error.Set(span, true)
96+
span.LogFields(otlog.Error(err))
97+
}
98+
span.Finish()
99+
}()
100+
101+
zr, err := gzip.NewReader(r)
102+
if err != nil {
103+
return nil, err
104+
}
105+
defer zr.Close()
106+
dec := gob.NewDecoder(zr)
107+
err = dec.Decode(&symbols)
108+
return symbols, err
109+
}

0 commit comments

Comments
 (0)