Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: allow URL and local files for Store locations #985

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
feat: allow URL and local files for Store locations
Enable local files and remote files over HTTP(S) as store locations for
the Function and Template stores.

This enables the following commands to work as expected

```sh
faas-cli store list --url $HOME/Downloads/functions.json
faas-cli store list --url ~/Downloads/functions.json
faas-cli store list --url https://internal/organization.com/openfaas/functions.json
faas-cli template store list --url $HOME/Downloads/templates.json
faas-cli template store list --url ~/Downloads/templates.json
faas-cli template store list --url
https://internal.organization.com/openfaas/templates.json
```

Additional locations such as ipfs, blob storage (s3, Google Cloud Storge, etc)
and other could also be supported via a small patch to the
`ReadJSON` implementation.

Signed-off-by: Lucas Roesler <[email protected]>
LucasRoesler committed Oct 29, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit c9c980edd0a315dc24305f6052ab9979ac5feaa5
50 changes: 2 additions & 48 deletions commands/store.go
Original file line number Diff line number Diff line change
@@ -4,14 +4,8 @@
package commands

import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"

"github.com/openfaas/faas-cli/proxy"
storeV2 "github.com/openfaas/faas-cli/schema/store/v2"
"github.com/spf13/cobra"
)
@@ -34,12 +28,13 @@ var (
const (
defaultStore = "https://raw.githubusercontent.com/openfaas/store/master/functions.json"
maxDescriptionLen = 40
storeAddressDoc = `Alternative path to Function Store metadata. It may be an http(s) URL or a local path to a JSON file.`
)

var platformValue string

func init() {
storeCmd.PersistentFlags().StringVarP(&storeAddress, "url", "u", defaultStore, "Alternative Store URL starting with http(s)://")
storeCmd.PersistentFlags().StringVarP(&storeAddress, "url", "u", defaultStore, storeAddressDoc)
storeCmd.PersistentFlags().StringVarP(&platformValue, "platform", "p", Platform, "Target platform for store")

faasCmd.AddCommand(storeCmd)
@@ -51,47 +46,6 @@ var storeCmd = &cobra.Command{
Long: "Allows browsing and deploying OpenFaaS functions from a store",
}

func storeList(store string) ([]storeV2.StoreFunction, error) {

var storeData storeV2.Store

store = strings.TrimRight(store, "/")

timeout := 60 * time.Second
tlsInsecure := false

client := proxy.MakeHTTPClient(&timeout, tlsInsecure)

res, err := client.Get(store)
if err != nil {
return nil, fmt.Errorf("cannot connect to OpenFaaS store at URL: %s", store)
}

if res.Body != nil {
defer res.Body.Close()
}

switch res.StatusCode {
case http.StatusOK:
bytesOut, err := io.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store)
}

jsonErr := json.Unmarshal(bytesOut, &storeData)
if jsonErr != nil {
return nil, fmt.Errorf("cannot parse result from OpenFaaS store at URL: %s\n%s", store, jsonErr.Error())
}
default:
bytesOut, err := io.ReadAll(res.Body)
if err == nil {
return nil, fmt.Errorf("server returned unexpected status code: %d - %s", res.StatusCode, string(bytesOut))
}
}

return storeData.Functions, nil
}

func filterStoreList(functions []storeV2.StoreFunction, platform string) []storeV2.StoreFunction {
var filteredList []storeV2.StoreFunction

2 changes: 1 addition & 1 deletion commands/store_deploy.go
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ func preRunEStoreDeploy(cmd *cobra.Command, args []string) error {

func runStoreDeploy(cmd *cobra.Command, args []string) error {
targetPlatform := getTargetPlatform(platformValue)
storeItems, err := storeList(storeAddress)
storeItems, err := proxy.FunctionStoreList(storeAddress)
if err != nil {
return err
}
3 changes: 2 additions & 1 deletion commands/store_describe.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (
"text/tabwriter"

"github.com/mitchellh/go-wordwrap"
"github.com/openfaas/faas-cli/proxy"
storeV2 "github.com/openfaas/faas-cli/schema/store/v2"
"github.com/spf13/cobra"
)
@@ -33,7 +34,7 @@ func runStoreDescribe(cmd *cobra.Command, args []string) error {
}

targetPlatform := getTargetPlatform(platformValue)
storeItems, err := storeList(storeAddress)
storeItems, err := proxy.FunctionStoreList(storeAddress)
if err != nil {
return err
}
3 changes: 2 additions & 1 deletion commands/store_list.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (
"strings"
"text/tabwriter"

"github.com/openfaas/faas-cli/proxy"
storeV2 "github.com/openfaas/faas-cli/schema/store/v2"
"github.com/spf13/cobra"
)
@@ -33,7 +34,7 @@ var storeListCmd = &cobra.Command{
func runStoreList(cmd *cobra.Command, args []string) error {
targetPlatform := getTargetPlatform(platformValue)

storeList, err := storeList(storeAddress)
storeList, err := proxy.FunctionStoreList(storeAddress)
if err != nil {
return err
}
2 changes: 1 addition & 1 deletion commands/template_store_describe.go
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ import (
)

func init() {
templateStoreDescribeCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates")
templateStoreDescribeCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc)

templateStoreCmd.AddCommand(templateStoreDescribeCmd)
}
43 changes: 6 additions & 37 deletions commands/template_store_list.go
Original file line number Diff line number Diff line change
@@ -6,24 +6,22 @@ package commands
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"

"net/http"
"os"
"sort"
"strings"
"text/tabwriter"
"time"

"github.com/openfaas/faas-cli/proxy"
"github.com/spf13/cobra"
)

const (
// DefaultTemplatesStore is the URL where the official store can be found
DefaultTemplatesStore = "https://raw.githubusercontent.com/openfaas/store/master/templates.json"
mainPlatform = "x86_64"
templateStoreDoc = `Alternative path to the template store metadata. It may be an http(s) URL or a local path to a JSON file.`
)

var (
@@ -35,7 +33,7 @@ var (

func init() {
templateStoreListCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Shows additional language and platform")
templateStoreListCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates")
templateStoreListCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc)
templateStoreListCmd.Flags().StringVarP(&inputPlatform, "platform", "p", mainPlatform, "Shows the platform if the output is verbose")
templateStoreListCmd.Flags().BoolVarP(&recommended, "recommended", "r", false, "Shows only recommended templates")
templateStoreListCmd.Flags().BoolVarP(&official, "official", "o", false, "Shows only official templates")
@@ -107,42 +105,13 @@ func runTemplateStoreList(cmd *cobra.Command, args []string) error {
}

func getTemplateInfo(repository string) ([]TemplateInfo, error) {
req, reqErr := http.NewRequest(http.MethodGet, repository, nil)
if reqErr != nil {
return nil, fmt.Errorf("error while trying to create request to take template info: %s", reqErr.Error())
}

reqContext, cancel := context.WithTimeout(req.Context(), 5*time.Second)
defer cancel()
req = req.WithContext(reqContext)

client := http.DefaultClient
res, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("error while requesting template list: %s", err.Error())
}

if res.Body == nil {
return nil, fmt.Errorf("error empty response body from: %s", templateStoreURL)
}
defer res.Body.Close()

if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code wanted: %d got: %d", http.StatusOK, res.StatusCode)
}

body, err := io.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("error while reading response: %s", err.Error())
}

templatesInfo := []TemplateInfo{}
if err := json.Unmarshal(body, &templatesInfo); err != nil {
return nil, fmt.Errorf("can't unmarshal text: %s", err.Error())
err := proxy.ReadJSON(context.TODO(), repository, &templatesInfo)
if err != nil {
return nil, fmt.Errorf("cannot read templates info from: %s", repository)
}

sortTemplates(templatesInfo)

return templatesInfo, nil
}

2 changes: 1 addition & 1 deletion commands/template_store_pull.go
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ import (
)

func init() {
templateStorePullCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, "Use as alternative store for templates")
templateStorePullCmd.PersistentFlags().StringVarP(&templateStoreURL, "url", "u", DefaultTemplatesStore, templateStoreDoc)
templatePull, _, _ := faasCmd.Find([]string{"template", "pull"})
templateStoreCmd.PersistentFlags().AddFlagSet(templatePull.Flags())

100 changes: 81 additions & 19 deletions proxy/function_store.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package proxy

import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/user"
"strings"
"time"

@@ -23,36 +26,95 @@ func FunctionStoreList(store string) ([]v2.StoreFunction, error) {

store = strings.TrimRight(store, "/")

err := ReadJSON(context.TODO(), store, &storeResults)
if err != nil {
return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store)
}

return storeResults.Functions, nil
}

// ReadJSON reads a JSON file from a URL or local file
func ReadJSON(ctx context.Context, location string, dest interface{}) error {
var body io.ReadCloser
var err error

timeout := 60 * time.Second
tlsInsecure := false

client := MakeHTTPClient(&timeout, tlsInsecure)
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

res, err := client.Get(store)
if err != nil {
return nil, fmt.Errorf("cannot connect to OpenFaaS store at URL: %s", store)
}
scheme := determineScheme(location)
switch scheme {
case "http", "https":
client := MakeHTTPClient(&timeout, tlsInsecure)

if res.Body != nil {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
if err != nil {
return fmt.Errorf("cannot create request to: %s", location)
}

res, err := client.Do(req)
if err != nil {
return fmt.Errorf("cannot connect to: %s", location)
}
defer res.Body.Close()
}

switch res.StatusCode {
case http.StatusOK:
bytesOut, err := io.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
return fmt.Errorf("server returned unexpected status code: %d", res.StatusCode)
}

body = res.Body
case "file":
location, err = expandTilde(location)
if err != nil {
return nil, fmt.Errorf("cannot read result from OpenFaaS store at URL: %s", store)
return err
}

jsonErr := json.Unmarshal(bytesOut, &storeResults)
if jsonErr != nil {
return nil, fmt.Errorf("cannot parse result from OpenFaaS store at URL: %s\n%s", store, jsonErr.Error())
body, err = os.Open(location)
if err != nil {
return fmt.Errorf("cannot read file: %s", location)
}

// Add more schemes such as s3:// or gs://
default:
bytesOut, err := io.ReadAll(res.Body)
if err == nil {
return nil, fmt.Errorf("server returned unexpected status code: %d - %s", res.StatusCode, string(bytesOut))
}
return fmt.Errorf("unsupported scheme: %s", scheme)
}
return storeResults.Functions, nil

if body != nil {
defer body.Close()
}

data, err := io.ReadAll(body)
if err != nil {
return fmt.Errorf("cannot read data from: %s", location)
}

return json.Unmarshal(data, dest)
}

func determineScheme(location string) string {
location = strings.ToLower(location)
if strings.HasPrefix(location, "http://") {
return "http"
}
if strings.HasPrefix(location, "https://") {
return "https"
}
return "file"
}

// expandTilde expands a path with a leading tilde to the home directory
func expandTilde(location string) (string, error) {
if !strings.HasPrefix(location, "~") {
return location, nil
}

usr, err := user.Current()
if err != nil {
return "", err
}

return strings.Replace(location, "~", usr.HomeDir, 1), nil
}
Loading