Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't commit assets #199

Merged
merged 6 commits into from
Mar 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,6 @@ typings/
Makefile.main
.ci
bin

# static included into go binary on release
/server/asset/asset.go
10 changes: 8 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ jobs:
- make lint
- make test-frontend
- make test-coverage
- name: 'Integration Tests'
stage: test
script:
- make packages
- ./build/bin/bblfsh-web &
- curl 127.0.0.1:9999 | grep 'Babelfish Web'
- name: 'Release'
stage: release
script:
Expand All @@ -51,10 +57,10 @@ jobs:
script:
- make packages
- DOCKER_PUSH_MASTER=true make docker-push
#HELM_RELEASE name is also harcoded in the .helm_staging file in order to use the AntiAffinity Rules.
#HELM_RELEASE name is also harcoded in the .helm_staging file in order to use the AntiAffinity Rules.
- HELM_RELEASE=bblfsh-web HELM_CHART=bblfsh-web K8S_NAMESPACE=default HELM_ARGS="--repo https://src-d.github.io/charts/ --version 0.8.0 --set image.tag=dev-$(git rev-parse --short HEAD)-dirty -f .helm_staging.yml" make deploy
- name: 'Deploy to production'
stage: release-helm
script:
#HELM_RELEASE name is also harcoded in the .helm_production file in order to use the AntiAffinity Rules.
#HELM_RELEASE name is also harcoded in the .helm_production file in order to use the AntiAffinity Rules.
- B64_CA_CRT=$B64_CA_CRT_PROD SERVICE_ACCOUNT_TOKEN=$SERVICE_ACCOUNT_TOKEN_PROD CLUSTER_ENDPOINT=$CLUSTER_ENDPOINT_PROD HELM_RELEASE=bblfsh-web HELM_CHART=bblfsh-web K8S_NAMESPACE=default HELM_ARGS="--repo https://src-d.github.io/charts/ --version 0.8.0 --set image.tag=$TRAVIS_TAG -f .helm_production.yml" make deploy
39 changes: 26 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,42 @@
PROJECT = web
COMMANDS = server/cmd/bblfsh-web
DEPENDENCIES = \
github.com/jteeuwen/go-bindata
github.com/mjibson/esc
DEPENDENCIES_DIRECTORY := ./vendor

# Including ci Makefile
CI_REPOSITORY ?= https://github.com/src-d/ci.git
CI_BRANCH ?= v1
CI_PATH ?= .ci
MAKEFILE := $(CI_PATH)/Makefile.main
$(MAKEFILE):
git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH);
-include $(MAKEFILE)

# Environment
BASE_PATH := $(shell pwd)
ASSETS_PATH := $(BASE_PATH)/build

# Tools
YARN = yarn
REMOVE = rm -rf
BINDATA = go-bindata
ESC_BIN := esc
GODEP = dep

SERVER_URL ?= api
BBLFSH_PORT ?= 9432
API_PORT ?= 9999

# Including ci Makefile
CI_REPOSITORY ?= https://github.com/src-d/ci.git
CI_BRANCH ?= v1
CI_PATH ?= .ci
MAKEFILE := $(CI_PATH)/Makefile.main
$(MAKEFILE):
git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH);
-include $(MAKEFILE)
WITH_STATIC_TAG := with_static

# Override Makefile.main defaults for arguments to be used in `go` commands.
GO_BUILD_ARGS := -ldflags "$(LD_FLAGS)" -tags "$(WITH_STATIC_TAG)"

# Environment and arguments to use in `go run` calls.
GO_RUN_ARGS += -tags "$(WITH_STATIC_TAG)"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this didn't came from lookout-web

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in lookout-web it looks like this: https://github.com/src-d/lookout/blob/master/Makefile.web#L69
same-same

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

kind of same functionality, but subtle different code (ie. GO_RUN_ARGS does not appear in that Makefile.web)
But as I said, not blocker at all, so we can merge as it is.
(I also locally tested, and worked like a charm)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah. Maybe we can actually extract go-handler part & Makefile somewhere and then just re-use in our projects. @carlosms wdyt? Shall I create an issue for that?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1
I've been proposing that since years 🎉

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sounds good to me. Let's open that issue :)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


GOCMD = go
GORUN = $(GOCMD) run $(GO_RUN_ARGS)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not in lookout-web


dependencies-frontend: dependencies
$(YARN) install
Expand All @@ -41,12 +52,13 @@ fix-lint-errors: dependencies-frontend
$(YARN) fix-lint-errors

assets:
mkdir -p ./server/asset; \
chmod -R go=r $(ASSETS_PATH); \
$(BINDATA) \
$(ESC_BIN) \
-pkg asset \
-o ./server/asset/asset.go \
-prefix $(BASE_PATH) \
$(ASSETS_PATH)/...
$(ASSETS_PATH)

front-build: dependencies-frontend
GENERATE_SOURCEMAP=false REACT_APP_SERVER_URL=$(SERVER_URL) $(YARN) build
Expand All @@ -58,5 +70,6 @@ validate-commit: fix-lint-errors no-changes-in-commit

## Compiles web client assets, and serve the web client through its API
serve:
$(MAKE) -C . front-build
$(MAKE) -C . assets SERVER_URL=http://0.0.0.0:$(API_PORT)/api
go run server/cmd/bblfsh-web/* -bblfsh-addr=0.0.0.0:$(BBLFSH_PORT) -addr=:$(API_PORT)
$(GORUN) server/cmd/bblfsh-web/* -bblfsh-addr=0.0.0.0:$(BBLFSH_PORT) -addr=:$(API_PORT)
429 changes: 0 additions & 429 deletions server/asset/asset.go

This file was deleted.

42 changes: 5 additions & 37 deletions server/cmd/bblfsh-web/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,11 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"time"

"github.com/bblfsh/web/server"
"github.com/bblfsh/web/server/asset"
"github.com/gin-gonic/gin"
"github.com/rs/cors"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -55,27 +51,12 @@ func main() {
r.Use(gin.RecoveryWithWriter(w))
r.Use(gin.LoggerWithWriter(w))

dir, err := mountAssets()
if err != nil {
logrus.Fatalf("unable to mount assets: %s", err)
}

assets, err := asset.AssetDir("build")
if err != nil {
logrus.Fatalf("cannot list assets: %s", err)
}

for _, a := range assets {
if a != "static" {
r.StaticFile("/"+a, filepath.Join(dir, a))
}
}
indexPath := filepath.Join(dir, "index.html")
r.StaticFile("/", indexPath)
r.Static("/static", filepath.Join(dir, "static"))
server.Mount(s, r.Group("/api"))
// we handle urls on frontend
r.NoRoute(func(c *gin.Context) { c.File(indexPath) })

// TODO(@smacker): add configuration for ServerURL and FooterHTML
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is functionality brought from the code c&p from lookout, but we can't use yet in bblfsh web, right?
Should we have an issue for this, to implement it later?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

static := server.NewStatic("/build", "", "")
// use static http.Handler for unknown urls, because we do routing on frontend
r.NoRoute(func(c *gin.Context) { static.ServeHTTP(c.Writer, c.Request) })

logrus.WithField("addr", addr).Info("starting REST server")

Expand All @@ -92,19 +73,6 @@ func main() {
}
}

func mountAssets() (string, error) {
dir, err := ioutil.TempDir(os.TempDir(), "bblfsh-web-assets")
if err != nil {
return "", err
}

if err := asset.RestoreAssets(dir, "build"); err != nil {
return "", err
}

return filepath.Join(dir, "build"), nil
}

func withCORS(handler http.Handler) http.Handler {
cors := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
Expand Down
103 changes: 103 additions & 0 deletions server/static_files.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// +build with_static

package server

import (
"bytes"
"encoding/base64"
"encoding/json"
"io/ioutil"
"net/http"
"strings"

"github.com/bblfsh/web/server/asset"
)

const (
staticDirName = "static"
indexFileName = "/index.html"

serverValuesPlaceholder = "window.REPLACE_BY_SERVER"
footerPlaceholder = `<div class="invisible-footer"></div>`
)

// Static contains handlers to serve static using esc
type Static struct {
fs http.FileSystem
options options
footerHTML []byte
}

// NewStatic creates new Static
func NewStatic(dir, serverURL string, footerHTML string) *Static {
var footerBytes []byte
if footerHTML != "" {
// skip incorrect base64
footerBytes, _ = base64.StdEncoding.DecodeString(footerHTML)
}

return &Static{
fs: asset.Dir(false, dir),
options: options{
ServerURL: serverURL,
},
footerHTML: footerBytes,
}
}

// struct which will be marshalled and exposed to frontend
type options struct {
ServerURL string `json:"SERVER_URL"`
}

// ServeHTTP serves any static file from static directory or fallbacks on index.hml
func (s *Static) ServeHTTP(w http.ResponseWriter, r *http.Request) {
_, err := s.fs.Open(r.URL.Path)
if err != nil {
if strings.HasPrefix(r.URL.Path, staticDirName) {
http.NotFound(w, r)
return
}

s.serveIndexHTML(nil)(w, r)
return
}

http.FileServer(s.fs).ServeHTTP(w, r)
}

// serveIndexHTML serves index.html file
func (s *Static) serveIndexHTML(initialState interface{}) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
f, err := s.fs.Open(indexFileName)
if err != nil {
http.NotFound(w, r)
return
}

b, err := ioutil.ReadAll(f)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}

options := s.options
bData, err := json.Marshal(options)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

b = bytes.Replace(b, []byte(serverValuesPlaceholder), bData, 1)
b = bytes.Replace(b, []byte(footerPlaceholder), s.footerHTML, 1)

w.Header().Add("Cache-Control", "no-cache, no-store, must-revalidate")

info, err := f.Stat()
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}

http.ServeContent(w, r, info.Name(), info.ModTime(), bytes.NewReader(b))
}
}
25 changes: 25 additions & 0 deletions server/static_files_dev.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// +build !with_static

package server

import (
"net/http"
)

// Static without the 'with_static' tag contains a placeholder handler
// that returns 'not implemented'
type Static struct{}

// NewStatic creates new Static
func NewStatic(dir, serverURL string, footerHTML string) *Static {
return &Static{}
}

// ServeHTTP serves any static file from static directory or fallbacks on index.hml
func (s *Static) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w,
"Frontend assets are only available when using 'make build' or 'make serve'",
http.StatusNotImplemented)

return
}