Skip to content

Commit 34c8b2a

Browse files
barelyhumanbarelyhuman
authored
Feat: Caching (#18)
* move minio to a seperate file * feat: rudimentary impl of from cache * ci: update versions * feat: add storage cleaning job * docs: update readme for self hosting config * docs: typo
1 parent 020a81e commit 34c8b2a

File tree

10 files changed

+276
-225
lines changed

10 files changed

+276
-225
lines changed

‎.docker-compose-with-storage.yml

-68
This file was deleted.

‎.env.example

+8-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
MINIO_ROOT_USER="" # eg: minioadmin
2-
MINIO_ROOT_PASSWORD="" # eg: "minioadmin"
3-
BUCKET_NAME="" # eg: "binaries"
1+
STORAGE_ENABLED=
2+
STORAGE_CLIENT_ID=
3+
STORAGE_CLIENT_SECRET=
4+
STORAGE_ENDPOINT=
5+
STORAGE_BUCKET=
46
ORIGIN_URL="" # eg: "http://localhost" depending on your nginx config changes
5-
MINIO_URL_PREFIX="" # eg: "http://localhost:9000" depending on your nginx config changes
67
GITHUB_TOKEN="" # required for github version resolutions
8+
9+
# uncomment the below to enable cache clearing based on the passed duration
10+
# CLEAR_CACHE_TIME="10m"

‎.github/workflows/go.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
runs-on: ubuntu-latest
1515
strategy:
1616
matrix:
17-
version: [1.16, 1.17, '1.20.0']
17+
version: ['1.19.0','1.21.5']
1818
steps:
1919
- uses: actions/checkout@v3
2020

‎cmd/goblin-api/main.go

+90-39
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"bytes"
45
"errors"
56
"flag"
67
"fmt"
@@ -11,17 +12,18 @@ import (
1112
"os"
1213
"path/filepath"
1314
"strings"
15+
"time"
1416

17+
"github.com/barelyhuman/go/env"
1518
"github.com/barelyhuman/goblin/build"
1619
"github.com/barelyhuman/goblin/resolver"
20+
"github.com/barelyhuman/goblin/storage"
1721
"github.com/joho/godotenv"
1822
)
1923

2024
var shTemplates *template.Template
2125
var serverURL string
22-
23-
// FIXME: Disabled storage and caching for initial version
24-
// var storageClient *storage.Storage
26+
var storageClient storage.Storage
2527

2628
func HandleRequest(rw http.ResponseWriter, req *http.Request) {
2729
path := req.URL.Path
@@ -40,6 +42,7 @@ func HandleRequest(rw http.ResponseWriter, req *http.Request) {
4042
}
4143

4244
if strings.HasPrefix(path, "/binary") {
45+
log.Print("handle binary")
4346
fetchBinary(rw, req)
4447
return
4548
}
@@ -63,20 +66,12 @@ func StartServer(port string) {
6366
}
6467
}
6568

66-
func envDefault(key string, def string) string {
67-
if s := os.Getenv(key); len(strings.TrimSpace(s)) == 0 {
68-
return def
69-
} else {
70-
return s
71-
}
72-
}
73-
7469
// TODO: cleanup code
7570
// TODO: move everything into their own interface/structs
7671
func main() {
7772

7873
envFile := flag.String("env", ".env", "path to read the env config from")
79-
portFlag := envDefault("PORT", "3000")
74+
portFlag := env.Get("PORT", "3000")
8075

8176
flag.Parse()
8277

@@ -88,19 +83,64 @@ func main() {
8883
}
8984

9085
shTemplates = template.Must(template.ParseGlob("templates/*"))
91-
serverURL = envDefault("ORIGIN_URL", "http://localhost:3000")
86+
serverURL = env.Get("ORIGIN_URL", "http://localhost:"+portFlag)
9287

93-
// FIXME: Disabled storage and caching for initial version
94-
// storageClient = &storage.Storage{}
95-
// storageClient.BucketName = os.Getenv("BUCKET_NAME")
96-
// err := storageClient.Connect()
97-
// if err != nil {
98-
// log.Fatal(err)
99-
// }
88+
if isStorageEnabled() {
89+
storageClient = storage.NewAWSStorage(env.Get("STORAGE_BUCKET", "goblin-cache"))
90+
err := storageClient.Connect()
91+
if err != nil {
92+
log.Fatal(err)
93+
}
94+
}
10095

96+
clearStorageBackgroundJob()
10197
StartServer(portFlag)
10298
}
10399

100+
func clearStorageBackgroundJob() {
101+
cacheHoldEnv := env.Get("CLEAR_CACHE_TIME", "")
102+
if len(cacheHoldEnv) == 0 {
103+
return
104+
}
105+
106+
cacheHoldDuration, _ := time.ParseDuration(cacheHoldEnv)
107+
108+
cleaner := func(storageClient storage.Storage) {
109+
log.Println("Cleaning Cached Storage Object")
110+
objects := storageClient.ListObjects()
111+
for _, obj := range objects {
112+
objExpiry := obj.LastModified.Add(cacheHoldDuration)
113+
if time.Now().Equal(objExpiry) || time.Now().After(objExpiry) {
114+
storageClient.RemoveObject(obj.Key)
115+
}
116+
}
117+
}
118+
119+
ticker := time.NewTicker(cacheHoldDuration)
120+
quit := make(chan struct{})
121+
122+
go func() {
123+
for {
124+
select {
125+
case <-ticker.C:
126+
cleaner(storageClient)
127+
case <-quit:
128+
ticker.Stop()
129+
return
130+
}
131+
}
132+
}()
133+
}
134+
135+
func isStorageEnabled() bool {
136+
useStorageEnv := env.Get("STORAGE_ENABLED", "false")
137+
useStorage := false
138+
if useStorageEnv == "true" {
139+
useStorage = true
140+
}
141+
return useStorage
142+
}
143+
104144
func normalizePackage(pkg string) string {
105145
// strip leading protocol
106146
pkg = strings.Replace(pkg, "https://", "", 1)
@@ -228,38 +268,49 @@ func fetchBinary(rw http.ResponseWriter, req *http.Request) {
228268
Module: mod,
229269
}
230270

231-
// TODO: check the storage for existing binary for the module
232-
// and return from the storage instead
233-
234271
immutable(rw)
235272

236-
// FIXME: Disabled storage and caching for initial version
237-
// var buf bytes.Buffer
238-
// err := bin.WriteBuild(io.MultiWriter(rw, &buf))
273+
artifactName := constructArtifactName(bin)
239274

240-
err := bin.WriteBuild(io.MultiWriter(rw))
275+
if isStorageEnabled() && storageClient.HasObject(artifactName) {
276+
url, _ := storageClient.GetSignedURL(artifactName)
277+
log.Println("From cache")
278+
http.Redirect(rw, req, url, http.StatusSeeOther)
279+
return
280+
}
281+
282+
var buf bytes.Buffer
283+
err := bin.WriteBuild(io.MultiWriter(rw, &buf))
241284

242285
if err != nil {
243286
rw.WriteHeader(http.StatusInternalServerError)
244287
fmt.Fprint(rw, err.Error())
245288
return
246289
}
247290

291+
if isStorageEnabled() {
292+
err = storageClient.Upload(
293+
artifactName,
294+
buf,
295+
)
296+
297+
if err != nil {
298+
log.Println("Failed to upload", err)
299+
}
300+
}
301+
248302
err = bin.Cleanup()
249303
if err != nil {
250304
log.Println("cleaning binary build", err)
251305
}
306+
}
252307

253-
// FIXME: Disabled storage and caching for initial version
254-
// err = storageClient.Upload(bin.Module, bin.Dest)
255-
// if err != nil {
256-
// fmt.Fprint(rw, err.Error())
257-
// return
258-
// }
259-
260-
// url, err := storageClient.GetSignedURL(bin.Module, bin.Name)
261-
// if err != nil {
262-
// fmt.Fprint(rw, err.Error())
263-
// return
264-
// }
308+
func constructArtifactName(bin *build.Binary) string {
309+
var artifactName strings.Builder
310+
artifactName.Write([]byte(bin.Name))
311+
artifactName.Write([]byte("-"))
312+
artifactName.Write([]byte(bin.OS))
313+
artifactName.Write([]byte("-"))
314+
artifactName.Write([]byte(bin.Arch))
315+
return artifactName.String()
265316
}

‎docker-compose.yml

+24-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,25 @@ services:
1616
- "3000:3000"
1717
volumes:
1818
- "./:/usr/src/app"
19+
profiles: [app]
20+
21+
minio:
22+
image: "minio/minio:RELEASE.2024-05-01T01-11-10Z"
23+
container_name: goblin_minio
24+
command: 'server --console-address ":9001" /home/files'
25+
healthcheck:
26+
test: ["CMD", "curl", "-I", "http://127.0.0.1:9000/minio/health/live"]
27+
interval: 30s
28+
timeout: 20s
29+
retries: 3
30+
env_file:
31+
- .env
32+
expose:
33+
- "9000:9000"
34+
- "9001:9001"
35+
volumes:
36+
- "goblin:/data"
37+
profiles: [storage]
1938

2039
nginx:
2140
container_name: goblin_nginx
@@ -27,4 +46,8 @@ services:
2746
- "80:80"
2847
- "9000:9000"
2948
volumes:
30-
- "./nginx.conf:/etc/nginx/nginx.conf:ro"
49+
- "./nginx.conf:/etc/nginx/nginx.conf:ro"
50+
profiles: [app]
51+
52+
volumes:
53+
goblin:

‎go.mod

+4-13
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,26 @@ go 1.19
44

55
require (
66
github.com/Masterminds/semver v1.5.0
7+
github.com/barelyhuman/go v0.2.2
78
github.com/google/go-github/v53 v53.2.0
89
github.com/joho/godotenv v1.5.1
9-
github.com/minio/minio-go/v7 v7.0.66
10+
github.com/minio/minio-go v6.0.14+incompatible
1011
github.com/tj/go-semver v1.0.0
1112
golang.org/x/oauth2 v0.16.0
1213
)
1314

1415
require (
1516
github.com/ProtonMail/go-crypto v1.0.0 // indirect
1617
github.com/cloudflare/circl v1.3.7 // indirect
17-
github.com/dustin/go-humanize v1.0.1 // indirect
18+
github.com/go-ini/ini v1.67.0 // indirect
1819
github.com/golang/protobuf v1.5.3 // indirect
1920
github.com/google/go-querystring v1.1.0 // indirect
20-
github.com/google/uuid v1.6.0 // indirect
21-
github.com/json-iterator/go v1.1.12 // indirect
22-
github.com/klauspost/compress v1.17.5 // indirect
23-
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
24-
github.com/minio/md5-simd v1.1.2 // indirect
25-
github.com/minio/sha256-simd v1.0.1 // indirect
26-
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
27-
github.com/modern-go/reflect2 v1.0.2 // indirect
28-
github.com/rs/xid v1.5.0 // indirect
29-
github.com/sirupsen/logrus v1.9.3 // indirect
21+
github.com/mitchellh/go-homedir v1.1.0 // indirect
3022
github.com/stretchr/testify v1.8.1 // indirect
3123
golang.org/x/crypto v0.18.0 // indirect
3224
golang.org/x/net v0.20.0 // indirect
3325
golang.org/x/sys v0.16.0 // indirect
3426
golang.org/x/text v0.14.0 // indirect
3527
google.golang.org/appengine v1.6.8 // indirect
3628
google.golang.org/protobuf v1.32.0 // indirect
37-
gopkg.in/ini.v1 v1.67.0 // indirect
3829
)

0 commit comments

Comments
 (0)