From 93a9ede63d56f2e18b023d26120e23d557341c26 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Fri, 19 Aug 2022 16:46:54 +0200 Subject: [PATCH 01/13] Add Column type to hold more information per table column as IsRequired aka NOT NULL --- internal/data/catalog.go | 9 ++++++++- internal/data/catalog_db.go | 5 +++-- internal/data/catalog_mock.go | 10 +++++----- internal/data/db_sql.go | 30 +++++++++++++++++++++++++----- internal/service/param.go | 2 +- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/internal/data/catalog.go b/internal/data/catalog.go index f687cf6d..b324f657 100644 --- a/internal/data/catalog.go +++ b/internal/data/catalog.go @@ -93,6 +93,13 @@ type QueryParam struct { TransformFuns []TransformFunction } +// Column holds metadata for column objects +type Column struct { + Index int + Type string + IsRequired bool +} + // Table holds metadata for table/view objects type Table struct { ID string @@ -106,7 +113,7 @@ type Table struct { Srid int Extent Extent Columns []string - DbTypes map[string]string + DbTypes map[string]Column JSONTypes []string ColDesc []string } diff --git a/internal/data/catalog_db.go b/internal/data/catalog_db.go index 764778e7..12a9dedc 100644 --- a/internal/data/catalog_db.go +++ b/internal/data/catalog_db.go @@ -341,7 +341,7 @@ func scanTable(rows pgx.Rows) *Table { // Since Go map order is random, list columns in array columns := make([]string, arrLen) jsontypes := make([]string, arrLen) - datatypes := make(map[string]string) + datatypes := make(map[string]Column) colDesc := make([]string, arrLen) for i := arrStart; i < arrLen; i++ { @@ -349,7 +349,8 @@ func scanTable(rows pgx.Rows) *Table { name := props.Elements[elmPos].String datatype := props.Elements[elmPos+1].String columns[i] = name - datatypes[name] = datatype + // TODO must find a way to compute IsRequired + datatypes[name] = Column{Index: i, Type: datatype, IsRequired: true} jsontypes[i] = toJSONTypeFromPG(datatype) colDesc[i] = props.Elements[elmPos+2].String } diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index ed100a73..48b6aee1 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -40,11 +40,11 @@ func CatMockInstance() *CatalogMock { func newCatalogMock() CatalogMock { // must be in synch with featureMock type propNames := []string{"prop_a", "prop_b", "prop_c", "prop_d"} - types := map[string]string{ - "prop_a": "text", - "prop_b": "int", - "prop_c": "text", - "prop_d": "int", + types := map[string]Column{ + "prop_a": Column{Index: 0, Type: "text", IsRequired: true}, + "prop_b": Column{Index: 0, Type: "int", IsRequired: true}, + "prop_c": Column{Index: 0, Type: "text", IsRequired: false}, + "prop_d": Column{Index: 0, Type: "int", IsRequired: false}, } jtypes := []string{"string", "number", "string", "number"} colDesc := []string{"Property A", "Property B", "Property C", "Property D"} diff --git a/internal/data/db_sql.go b/internal/data/db_sql.go index 35b10019..77fea684 100644 --- a/internal/data/db_sql.go +++ b/internal/data/db_sql.go @@ -130,7 +130,8 @@ const sqlFmtFeatures = "SELECT %v %v FROM \"%s\".\"%s\" %v %v %v %s;" func sqlFeatures(tbl *Table, param *QueryParam) (string, []interface{}) { geomCol := sqlGeomCol(tbl.GeometryColumn, tbl.Srid, param) - propCols := sqlColList(param.Columns, tbl.DbTypes, true) + + propCols := sqlColListFromColumnMap(param.Columns, tbl.DbTypes, true) bboxFilter := sqlBBoxFilter(tbl.GeometryColumn, tbl.Srid, param.Bbox, param.BboxCrs) attrFilter, attrVals := sqlAttrFilter(param.Filter) cqlFilter := sqlCqlFilter(param.FilterSql) @@ -144,7 +145,25 @@ func sqlFeatures(tbl *Table, param *QueryParam) (string, []interface{}) { // sqlColList creates a comma-separated column list, or blank if no columns // If addLeadingComma is true, a leading comma is added, for use when the target SQL has columns defined before -func sqlColList(names []string, dbtypes map[string]string, addLeadingComma bool) string { +func sqlColListFromColumnMap(names []string, dbtypes map[string]Column, addLeadingComma bool) string { + if len(names) == 0 { + return "" + } + + var cols []string + for _, col := range names { + colExpr := sqlColExpr(col, dbtypes[col].Type) + cols = append(cols, colExpr) + } + colsStr := strings.Join(cols, ",") + if addLeadingComma { + return ", " + colsStr + } + return colsStr +} + +// sqlColListFromPGTypeMap creates a comma-separated column list, or blank if no columns +func sqlColListFromStringMap(names []string, dbtypes map[string]string, addLeadingComma bool) string { if len(names) == 0 { return "" } @@ -186,7 +205,8 @@ const sqlFmtFeature = "SELECT %v %v FROM \"%s\".\"%s\" WHERE \"%v\" = $1 LIMIT 1 func sqlFeature(tbl *Table, param *QueryParam) string { geomCol := sqlGeomCol(tbl.GeometryColumn, tbl.Srid, param) - propCols := sqlColList(param.Columns, tbl.DbTypes, true) + + propCols := sqlColListFromColumnMap(param.Columns, tbl.DbTypes, true) sql := fmt.Sprintf(sqlFmtFeature, geomCol, propCols, tbl.Schema, tbl.Table, tbl.IDColumn) return sql } @@ -328,7 +348,7 @@ const sqlFmtGeomFunction = "SELECT %s %s FROM \"%s\".\"%s\"( %v ) %v %v %s;" func sqlGeomFunction(fn *Function, args map[string]string, propCols []string, param *QueryParam) (string, []interface{}) { sqlArgs, argVals := sqlFunctionArgs(fn, args) sqlGeomCol := sqlGeomCol(fn.GeometryColumn, SRID_UNKNOWN, param) - sqlPropCols := sqlColList(propCols, fn.Types, true) + sqlPropCols := sqlColListFromStringMap(propCols, fn.Types, true) //-- SRS of function output is unknown, so have to assume 4326 bboxFilter := sqlBBoxFilter(fn.GeometryColumn, SRID_4326, param.Bbox, param.BboxCrs) cqlFilter := sqlCqlFilter(param.FilterSql) @@ -343,7 +363,7 @@ const sqlFmtFunction = "SELECT %v FROM \"%s\".\"%s\"( %v ) %v %v %s;" func sqlFunction(fn *Function, args map[string]string, propCols []string, param *QueryParam) (string, []interface{}) { sqlArgs, argVals := sqlFunctionArgs(fn, args) - sqlPropCols := sqlColList(propCols, fn.Types, false) + sqlPropCols := sqlColListFromStringMap(propCols, fn.Types, false) cqlFilter := sqlCqlFilter(param.FilterSql) sqlWhere := sqlWhere(cqlFilter, "", "") sqlOrderBy := sqlOrderBy(param.SortBy) diff --git a/internal/service/param.go b/internal/service/param.go index 5529c8ce..91e92e8b 100644 --- a/internal/service/param.go +++ b/internal/service/param.go @@ -405,7 +405,7 @@ func parseTransformFun(def string) data.TransformFunction { } // parseFilter creates a filter list from applicable query parameters -func parseFilter(paramMap map[string]string, colNameMap map[string]string) []*data.PropertyFilter { +func parseFilter(paramMap map[string]string, colNameMap map[string]data.Column) []*data.PropertyFilter { var conds []*data.PropertyFilter for name, val := range paramMap { //log.Debugf("testing request param %v", name) From 6ba7bebff7e54bcb0d4775777e69bf2192e16ee6 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Fri, 19 Aug 2022 16:58:20 +0200 Subject: [PATCH 02/13] feat(CREATE): add json schema retrieval for create type from /collections/{id}/schema * update openapi paths description * split handler_test.go in 3: * handler_test.go: to test get operations * handler_common_test.go: common test functions * handler_post_test.go: to test post operations * fix bad column indexes in catalog_mock --- internal/api/api.go | 1 + internal/api/net.go | 9 ++ internal/api/openapi.go | 38 +++++++- internal/data/catalog_mock.go | 65 +++++++------- internal/service/handler.go | 94 ++++++++++++++++++++ internal/service/handler_common_test.go | 110 ++++++++++++++++++++++++ internal/service/handler_post_test.go | 67 +++++++++++++++ internal/service/handler_test.go | 54 +----------- 8 files changed, 353 insertions(+), 85 deletions(-) create mode 100644 internal/service/handler_common_test.go create mode 100644 internal/service/handler_post_test.go diff --git a/internal/api/api.go b/internal/api/api.go index 270dbe3b..246a8961 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -45,6 +45,7 @@ const ( ParamProperties = "properties" ParamSortBy = "sortby" ParamTransform = "transform" + ParamType = "type" OrderByDirSep = ":" OrderByDirD = "d" diff --git a/internal/api/net.go b/internal/api/net.go index 9a45961a..e1ff2603 100644 --- a/internal/api/net.go +++ b/internal/api/net.go @@ -26,6 +26,9 @@ const ( // ContentTypeGeoJSON ContentTypeGeoJSON = "application/geo+json" + // ContentTypeSchemaJSON + ContentTypeSchemaJSON = "application/schema+json" + // ContentTypeHTML ContentTypeHTML = "text/html" @@ -49,6 +52,9 @@ const ( // FormatText code and extension for Text FormatSVG = "svg" + + // FormatJSON code and extension for JSON + FormatSchemaJSON = "schema+json" ) // RequestedFormat gets the format for a request from extension or headers @@ -70,6 +76,9 @@ func RequestedFormat(r *http.Request) string { // Use Accept header if present hdrAccept := r.Header.Get("Accept") //fmt.Println("Accept:" + hdrAccept) + if strings.Contains(hdrAccept, ContentTypeSchemaJSON) { + return FormatSchemaJSON + } if strings.Contains(hdrAccept, ContentTypeHTML) { return FormatHTML } diff --git a/internal/api/openapi.go b/internal/api/openapi.go index 8e28c256..5a0bf7c3 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -211,6 +211,16 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { AllowEmptyValue: false, }, } + paramType := openapi3.ParameterRef{ + Value: &openapi3.Parameter{ + Name: "type", + Description: "Data schema type (create, update, etc.).", + In: "query", + Required: false, + Schema: &openapi3.SchemaRef{Value: openapi3.NewStringSchema()}, + AllowEmptyValue: false, + }, + } return &openapi3.Swagger{ OpenAPI: "3.0.0", Info: openapi3.Info{ @@ -342,7 +352,7 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { Responses: openapi3.Responses{ "200": &openapi3.ResponseRef{ Value: &openapi3.Response{ - Description: "GeoJSON Featuree Collection document containing data for features", + Description: "GeoJSON Feature Collection document containing data for features", /* // TODO: create schema for result? Content: openapi3.NewContentWithJSONSchemaRef( @@ -356,6 +366,32 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { }, }, }, + apiBase + "collections/{collectionId}/schema": &openapi3.PathItem{ + Summary: "Feature schema for collection", + Description: "Provides access to data representation (schema) for any features in specified collection", + Get: &openapi3.Operation{ + OperationID: "getCollectionSchema", + Parameters: openapi3.Parameters{ + ¶mCollectionID, + ¶mType, + }, + Responses: openapi3.Responses{ + "200": &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: "GeoJSON Feature Collection document containing data schema for specific type", + /* + // TODO: create schema for result? + Content: openapi3.NewContentWithJSONSchemaRef( + &openapi3.SchemaRef{ + Ref: "http://geojson.org/schema/FeatureSchema.json", + }, + ), + */ + }, + }, + }, + }, + }, apiBase + "collections/{collectionId}/items/{featureId}": &openapi3.PathItem{ Summary: "Single feature data from collection", Description: "Provides access to a single feature identitfied by {featureId} from the specified collection", diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index 48b6aee1..cef14e36 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -41,48 +41,51 @@ func newCatalogMock() CatalogMock { // must be in synch with featureMock type propNames := []string{"prop_a", "prop_b", "prop_c", "prop_d"} types := map[string]Column{ - "prop_a": Column{Index: 0, Type: "text", IsRequired: true}, - "prop_b": Column{Index: 0, Type: "int", IsRequired: true}, - "prop_c": Column{Index: 0, Type: "text", IsRequired: false}, - "prop_d": Column{Index: 0, Type: "int", IsRequired: false}, + "prop_a": {Index: 0, Type: "text", IsRequired: true}, + "prop_b": {Index: 1, Type: "int", IsRequired: true}, + "prop_c": {Index: 2, Type: "text", IsRequired: false}, + "prop_d": {Index: 3, Type: "int", IsRequired: false}, } jtypes := []string{"string", "number", "string", "number"} colDesc := []string{"Property A", "Property B", "Property C", "Property D"} layerA := &Table{ - ID: "mock_a", - Title: "Mock A", - Description: "This dataset contains mock data about A (9 points)", - Extent: Extent{Minx: -120, Miny: 40, Maxx: -74, Maxy: 50}, - Srid: 4326, - Columns: propNames, - DbTypes: types, - JSONTypes: jtypes, - ColDesc: colDesc, + ID: "mock_a", + Title: "Mock A", + Description: "This dataset contains mock data about A (9 points)", + Extent: Extent{Minx: -120, Miny: 40, Maxx: -74, Maxy: 50}, + Srid: 4326, + GeometryType: "Point", + Columns: propNames, + DbTypes: types, + JSONTypes: jtypes, + ColDesc: colDesc, } layerB := &Table{ - ID: "mock_b", - Title: "Mock B", - Description: "This dataset contains mock data about B (100 points)", - Extent: Extent{Minx: -75, Miny: 45, Maxx: -74, Maxy: 46}, - Srid: 4326, - Columns: propNames, - DbTypes: types, - JSONTypes: jtypes, - ColDesc: colDesc, + ID: "mock_b", + Title: "Mock B", + Description: "This dataset contains mock data about B (100 points)", + Extent: Extent{Minx: -75, Miny: 45, Maxx: -74, Maxy: 46}, + Srid: 4326, + GeometryType: "Point", + Columns: propNames, + DbTypes: types, + JSONTypes: jtypes, + ColDesc: colDesc, } layerC := &Table{ - ID: "mock_c", - Title: "Mock C", - Description: "This dataset contains mock data about C (10000 points)", - Extent: Extent{Minx: -120, Miny: 40, Maxx: -74, Maxy: 60}, - Srid: 4326, - Columns: propNames, - DbTypes: types, - JSONTypes: jtypes, - ColDesc: colDesc, + ID: "mock_c", + Title: "Mock C", + Description: "This dataset contains mock data about C (10000 points)", + Extent: Extent{Minx: -120, Miny: 40, Maxx: -74, Maxy: 60}, + Srid: 4326, + GeometryType: "Point", + Columns: propNames, + DbTypes: types, + JSONTypes: jtypes, + ColDesc: colDesc, } tableData := map[string][]*featureMock{} diff --git a/internal/service/handler.go b/internal/service/handler.go index c55b0b87..6ace8938 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -24,6 +24,7 @@ import ( "github.com/CrunchyData/pg_featureserv/internal/conf" "github.com/CrunchyData/pg_featureserv/internal/data" "github.com/CrunchyData/pg_featureserv/internal/ui" + "github.com/getkin/kin-openapi/openapi3" "github.com/gorilla/mux" ) @@ -58,6 +59,8 @@ func initRouter(basePath string) *mux.Router { addRoute(router, "/collections/{id}/items", handleCollectionItems) addRoute(router, "/collections/{id}/items.{fmt}", handleCollectionItems) + addRoute(router, "/collections/{id}/schema", handleCollectionSchemas) + addRoute(router, "/collections/{id}/items/{fid}", handleItem) addRoute(router, "/collections/{id}/items/{fid}.{fmt}", handleItem) @@ -254,6 +257,45 @@ func handleCollection(w http.ResponseWriter, r *http.Request) *appError { } } +func handleCollectionSchemas(w http.ResponseWriter, r *http.Request) *appError { + // TODO: determine content from request header? + format := api.RequestedFormat(r) + urlBase := serveURLBase(r) + + //--- extract request parameters + name := getRequestVar(routeVarID, r) + tbl, err1 := catalogInstance.TableByName(name) + if err1 != nil { + return appErrorInternalFmt(err1, api.ErrMsgCollectionAccess, name) + } + if tbl == nil { + return appErrorNotFoundFmt(err1, api.ErrMsgCollectionNotFound, name) + } + + queryValues := r.URL.Query() + paramValues := extractSingleArgs(queryValues) + // --- type parameter + schemaType := parseString(paramValues, api.ParamType) + + ctx := r.Context() + switch format { + case api.FormatSchemaJSON: + { + switch schemaType { + case "create": + return writeCreateItemSchemaJSON(ctx, w, tbl, urlBase) + default: + return appErrorBadRequest(nil, fmt.Sprintf("Asked schema type %s not implemented!", schemaType)) + } + } + default: + { + return appErrorBadRequest(nil, fmt.Sprintf("Format %s not implemented!", format)) + } + } + +} + func handleCollectionItems(w http.ResponseWriter, r *http.Request) *appError { // TODO: determine content from request header? format := api.RequestedFormat(r) @@ -290,6 +332,58 @@ func handleCollectionItems(w http.ResponseWriter, r *http.Request) *appError { return nil } +func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table *data.Table, urlBase string) *appError { + // Feature schema skeleton + var featureInfoSchema openapi3.Schema = openapi3.Schema{ + Type: "object", + Required: []string{"type", "geometry", "properties"}, + Properties: map[string]*openapi3.SchemaRef{ + "id": {Value: &openapi3.Schema{Type: "number", Format: "long"}}, + "type": { + Value: &openapi3.Schema{ + Type: "string", + Default: "Feature", + }, + }, + "geometry": { + Value: &openapi3.Schema{ + Items: &openapi3.SchemaRef{ + Ref: fmt.Sprintf("https://geojson.org/schema/%v.json", table.GeometryType), + }, + }, + }, + "properties": { + Value: &openapi3.Schema{}, + }, + }, + } + featureInfoSchema.Description = table.Description + + props := featureInfoSchema.Properties["properties"].Value + props.Type = "object" + + // update required properties + var requiredTypes []string + for k, c := range table.DbTypes { + if c.IsRequired { + requiredTypes = append(requiredTypes, k) + } + } + props.Required = requiredTypes + + // update properties by their name and type + props.Properties = make(map[string]*openapi3.SchemaRef) + for k, v := range table.DbTypes { + props.Properties[k] = &openapi3.SchemaRef{ + Value: &openapi3.Schema{ + Type: v.Type, + }, + } + } + + return writeJSON(w, api.ContentTypeSchemaJSON, featureInfoSchema) +} + func writeItemsHTML(w http.ResponseWriter, tbl *data.Table, name string, query string, urlBase string) *appError { pathItems := api.PathCollectionItems(name) diff --git a/internal/service/handler_common_test.go b/internal/service/handler_common_test.go new file mode 100644 index 00000000..12c57ed0 --- /dev/null +++ b/internal/service/handler_common_test.go @@ -0,0 +1,110 @@ +package service + +/* + Copyright 2022 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// returns the all the body from an http response +func readBody(resp *httptest.ResponseRecorder) []byte { + body, _ := ioutil.ReadAll(resp.Body) + return body +} + +// do an http request to url with default method GET and expected status OK +func doRequest(t *testing.T, url string) *httptest.ResponseRecorder { + return doRequestStatus(t, url, http.StatusOK) +} + +// do an http request to url with default method GET and a specific expected status +func doRequestStatus(t *testing.T, url string, + statusExpected int) *httptest.ResponseRecorder { + req, err := http.NewRequest("GET", basePath+url, nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + // Check the status code + //fmt.Println("Status:", rr.Code) + if status := rr.Code; status != statusExpected { + t.Errorf("handler returned wrong status code: got %v want %v", + status, statusExpected) + } + return rr +} + +// do an http request to url with a specific method and specific expected status +func doRequestMethodStatus(t *testing.T, method string, url string, + data []byte, header http.Header, statusExpected int) *httptest.ResponseRecorder { + + req, err := http.NewRequest(method, basePath+url, bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + + for k, v := range header { + req.Header.Add(k, v[0]) + } + + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + // Check the status code + status := rr.Code + if status != statusExpected { + var bodyMsg string + body, errBody := ioutil.ReadAll(rr.Body) + if errBody == nil && len(body) != 0 { + bodyMsg = "Error: " + string(body) + } + + assert(t, + status == statusExpected, + fmt.Errorf("handler returned wrong status code: got %v want %v. %v", + status, statusExpected, bodyMsg).Error()) + } + return rr +} + +//---- testing utilities from https://github.com/benbjohnson/testing + +// assert fails the test if the condition is false. +func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { + if !condition { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) + tb.FailNow() + } +} + +// equals fails the test if exp is not equal to act. +func equals(tb testing.TB, exp, act interface{}, msg string) { + if !reflect.DeepEqual(exp, act) { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("%s:%d: %s - expected: %#v; got: %#v\n", filepath.Base(file), line, msg, exp, act) + tb.FailNow() + } +} diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go new file mode 100644 index 00000000..5069c300 --- /dev/null +++ b/internal/service/handler_post_test.go @@ -0,0 +1,67 @@ +package service + +/* + Copyright 2022 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "testing" + + "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/getkin/kin-openapi/openapi3" +) + +// checks swagger api contains get operation from collection schema +func TestApiContainsCollectionSchemas(t *testing.T) { + resp := doRequest(t, "/api") + body, _ := ioutil.ReadAll(resp.Body) + + var v openapi3.Swagger + json.Unmarshal(body, &v) + + equals(t, 11, len(v.Paths), "# api paths") + path := v.Paths.Find("/collections/{collectionId}/schema") + assert(t, path != nil, "schema path exists") + equals(t, "Provides access to data representation (schema) for any features in specified collection", path.Description, "schema path present") + equals(t, "getCollectionSchema", path.Get.OperationID, "schema path get present") + equals(t, 2, len(path.Get.Parameters), "schema path get present") + assert(t, path.Get.Parameters.GetByInAndName("path", "collectionId") != nil, "collectionId path parameter exists") + assert(t, path.Get.Parameters.GetByInAndName("query", "type") != nil, "type query parameter exists") +} + +// checks collection schema contains valid data description +func TestGetCollectionCreateSchema(t *testing.T) { + path := "/collections/mock_a/schema?type=create" + var header = make(http.Header) + header.Add("Accept", api.ContentTypeSchemaJSON) + + resp := doRequestMethodStatus(t, "GET", path, nil, header, http.StatusOK) + body, _ := ioutil.ReadAll(resp.Body) + + fmt.Println(string(body)) + + var fis openapi3.Schema + err := fis.UnmarshalJSON(body) + if err != nil { + t.Fatal(err) + } + + equals(t, "This dataset contains mock data about A (9 points)", fis.Description, "feature description") + equals(t, "https://geojson.org/schema/Point.json", fis.Properties["geometry"].Value.Items.Ref, "feature geometry") + equals(t, "prop_a", fis.Properties["properties"].Value.Required[0], "feature required a") + equals(t, "prop_b", fis.Properties["properties"].Value.Required[1], "feature required b") + equals(t, "Feature", fis.Properties["type"].Value.Default, "feature required b") +} diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index 1f705dd3..c68457f1 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -15,14 +15,9 @@ package service import ( "encoding/json" - "fmt" "io/ioutil" "net/http" - "net/http/httptest" "os" - "path/filepath" - "reflect" - "runtime" "strings" "testing" @@ -433,34 +428,6 @@ func TestHTMLFunction(t *testing.T) { //=================================================== -func readBody(resp *httptest.ResponseRecorder) []byte { - body, _ := ioutil.ReadAll(resp.Body) - return body -} - -func doRequest(t *testing.T, url string) *httptest.ResponseRecorder { - return doRequestStatus(t, url, http.StatusOK) -} - -func doRequestStatus(t *testing.T, url string, - statusExpected int) *httptest.ResponseRecorder { - req, err := http.NewRequest("GET", basePath+url, nil) - if err != nil { - t.Fatal(err) - } - - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - - // Check the status code - //fmt.Println("Status:", rr.Code) - if status := rr.Code; status != statusExpected { - t.Errorf("handler returned wrong status code: got %v want %v", - status, statusExpected) - } - return rr -} - func checkCollection(tb testing.TB, coll *api.CollectionInfo, name string, title string) { equals(tb, name, coll.Name, "Collection name") equals(tb, title, coll.Title, "Collection title") @@ -472,6 +439,7 @@ func checkCollection(tb testing.TB, coll *api.CollectionInfo, name string, title pathItems := path + "/items" checkLink(tb, coll.Links[2], api.RelItems, api.ContentTypeGeoJSON, urlBase+pathItems) } + func checkLink(tb testing.TB, link *api.Link, rel string, conType string, href string) { equals(tb, rel, link.Rel, "Link rel") equals(tb, conType, link.Type, "Link type") @@ -530,23 +498,3 @@ func checkFunction(t *testing.T, fun *data.Function) { } checkLink(t, v.Links[2], api.RelItems, itemsType, urlBase+path+"/items") } - -//---- testing utilities from https://github.com/benbjohnson/testing - -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() - } -} - -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}, msg string) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: %s - expected: %#v; got: %#v\n", filepath.Base(file), line, msg, exp, act) - tb.FailNow() - } -} From 5073a103f6accd7c66223d085eac8f20ff3fba54 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Tue, 30 Aug 2022 17:25:31 +0200 Subject: [PATCH 03/13] Apply fixes from golangci-lint --- internal/service/handler_post_test.go | 3 ++- internal/service/handler_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index 5069c300..85111009 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -30,7 +30,8 @@ func TestApiContainsCollectionSchemas(t *testing.T) { body, _ := ioutil.ReadAll(resp.Body) var v openapi3.Swagger - json.Unmarshal(body, &v) + errUnMarsh := json.Unmarshal(body, &v) + assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) equals(t, 11, len(v.Paths), "# api paths") path := v.Paths.Find("/collections/{collectionId}/schema") diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index c68457f1..1edef8e4 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -15,6 +15,7 @@ package service import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "os" From 62159d93288741064c3e69f8dae8e9ad7c2b3929 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Fri, 19 Aug 2022 17:01:40 +0200 Subject: [PATCH 04/13] feat(CREATE): add functionality to create new feature in a collection. --- internal/api/api.go | 1 + internal/api/net.go | 19 ++++++++ internal/data/catalog.go | 4 ++ internal/data/catalog_db.go | 4 ++ internal/data/catalog_mock.go | 10 +++++ internal/service/handler.go | 60 +++++++++++++++++++++++-- internal/service/handler_common_test.go | 20 +++------ internal/service/handler_post_test.go | 37 +++++++++++++++ internal/service/handler_test.go | 29 ++++++++++++ 9 files changed, 166 insertions(+), 18 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index 246a8961..36dce0e0 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -76,6 +76,7 @@ const ( ErrMsgCollectionNotFound = "Collection not found: %v" ErrMsgCollectionAccess = "Unable to access Collection: %v" ErrMsgFeatureNotFound = "Feature not found: %v" + ErrMsgCreateFeature = "Unable to create new feature in Collection: %v" ErrMsgLoadFunctions = "Unable to access Functions" ErrMsgFunctionNotFound = "Function not found: %v" ErrMsgFunctionAccess = "Unable to access Function: %v" diff --git a/internal/api/net.go b/internal/api/net.go index e1ff2603..5b6401c2 100644 --- a/internal/api/net.go +++ b/internal/api/net.go @@ -26,6 +26,9 @@ const ( // ContentTypeGeoJSON ContentTypeGeoJSON = "application/geo+json" + // ContentTypeGML + ContentTypeGML = "application/gml+xml" + // ContentTypeSchemaJSON ContentTypeSchemaJSON = "application/schema+json" @@ -55,6 +58,9 @@ const ( // FormatJSON code and extension for JSON FormatSchemaJSON = "schema+json" + + // FormatXML code and extension for XML/GML + FormatXML = "xml" ) // RequestedFormat gets the format for a request from extension or headers @@ -85,6 +91,19 @@ func RequestedFormat(r *http.Request) string { return FormatJSON } +// RequestedFormat gets the format for a request from extension or headers +func SentDataFormat(r *http.Request) string { + // Use ContentType header if present + hdrContentType := r.Header.Get("Content-Type") + if strings.Contains(hdrContentType, ContentTypeGeoJSON) { + return FormatJSON + } + if strings.Contains(hdrContentType, ContentTypeGML) { + return FormatXML + } + return FormatJSON +} + // PathStripFormat removes a format extension from a path func PathStripFormat(path string) string { if strings.HasSuffix(path, ".html") || strings.HasSuffix(path, ".json") { diff --git a/internal/data/catalog.go b/internal/data/catalog.go index b324f657..f522c7a1 100644 --- a/internal/data/catalog.go +++ b/internal/data/catalog.go @@ -47,6 +47,10 @@ type Catalog interface { // It returns an empty string if the table or feature does not exist TableFeature(ctx context.Context, name string, id string, param *QueryParam) (string, error) + // AddTableFeature returns the id of the new feature created in the table tableName + // using the JSON data to create the feature + AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) + Functions() ([]*Function, error) // FunctionByName returns the function with given name. diff --git a/internal/data/catalog_db.go b/internal/data/catalog_db.go index 12a9dedc..e015c113 100644 --- a/internal/data/catalog_db.go +++ b/internal/data/catalog_db.go @@ -235,6 +235,10 @@ func (cat *catalogDB) TableFeature(ctx context.Context, name string, id string, return features[0], nil } +func (cat *catalogDB) AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) { + panic("catalogDB::AddTableFeature unimplemented") +} + func (cat *catalogDB) refreshTables(force bool) { // TODO: refresh on timed basis? if force || isStartup { diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index cef14e36..5b987296 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -245,6 +245,10 @@ func (cat *CatalogMock) TableFeature(ctx context.Context, name string, id string return features[index].toJSON(propNames), nil } +func (cat *CatalogMock) AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) { + panic("CatalogMock::AddTableFeature unimplemented") +} + func (cat *CatalogMock) Functions() ([]*Function, error) { return cat.FunctionDefs, nil } @@ -269,6 +273,12 @@ func (cat *CatalogMock) FunctionData(ctx context.Context, name string, args map[ return nil, nil } +func (cat *CatalogMock) MakeFeatureMockPointAsJSON(id int, x float64, y float64) string { + feat := makeFeatureMockPoint(id, x, y) + propNames := cat.TableDefs[0].Columns + return feat.toJSON(propNames) +} + func makePointFeatures(extent Extent, nx int, ny int) []*featureMock { basex := extent.Minx basey := extent.Miny diff --git a/internal/service/handler.go b/internal/service/handler.go index 6ace8938..29357a9b 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -17,7 +17,9 @@ import ( "bytes" "context" "fmt" + "io/ioutil" "net/http" + "sort" "strings" "github.com/CrunchyData/pg_featureserv/internal/api" @@ -58,6 +60,7 @@ func initRouter(basePath string) *mux.Router { addRoute(router, "/collections/{id}/items", handleCollectionItems) addRoute(router, "/collections/{id}/items.{fmt}", handleCollectionItems) + addRouteWithMethod(router, "/collections/{id}/items", handleCreateCollectionItem, "POST") addRoute(router, "/collections/{id}/schema", handleCollectionSchemas) @@ -77,7 +80,11 @@ func initRouter(basePath string) *mux.Router { } func addRoute(router *mux.Router, path string, handler func(http.ResponseWriter, *http.Request) *appError) { - router.Handle(path, appHandler(handler)) + addRouteWithMethod(router, path, handler, "GET") +} + +func addRouteWithMethod(router *mux.Router, path string, handler func(http.ResponseWriter, *http.Request) *appError, method string) { + router.Handle(path, appHandler(handler)).Methods(method) } //nolint:unused @@ -296,6 +303,45 @@ func handleCollectionSchemas(w http.ResponseWriter, r *http.Request) *appError { } +func handleCreateCollectionItem(w http.ResponseWriter, r *http.Request) *appError { + urlBase := serveURLBase(r) + + //--- extract request parameters + name := getRequestVar(routeVarID, r) + + //--- check query parameters + queryValues := r.URL.Query() + paramValues := extractSingleArgs(queryValues) + if len(paramValues) != 0 { + return appErrorMsg(nil, "No parameter allowed", http.StatusBadRequest) + } + + //--- check feature availability + tbl, err1 := catalogInstance.TableByName(name) + if err1 != nil { + return appErrorInternalFmt(err1, api.ErrMsgCollectionAccess, name) + } + if tbl == nil { + return appErrorNotFoundFmt(err1, api.ErrMsgCollectionNotFound, name) + } + + //--- json body + body, errBody := ioutil.ReadAll(r.Body) + if errBody != nil || len(body) == 0 { + return appErrorInternalFmt(errBody, "Unable to read request body for Collection: %v", name) + } + + newId, err2 := catalogInstance.AddTableFeature(r.Context(), name, body) + if err2 != nil { + return appErrorInternalFmt(err2, api.ErrMsgCreateFeature, name) + } + + w.Header().Set("Location", fmt.Sprintf("%scollections/%s/items/%d", urlBase, name, newId)) + w.WriteHeader(http.StatusCreated) + w.Write([]byte("")) + return nil +} + func handleCollectionItems(w http.ResponseWriter, r *http.Request) *appError { // TODO: determine content from request header? format := api.RequestedFormat(r) @@ -363,12 +409,20 @@ func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table props.Type = "object" // update required properties + requiredTypeKeys := make([]string, 0, len(table.DbTypes)) + + for k := range table.DbTypes { + requiredTypeKeys = append(requiredTypeKeys, k) + } + sort.Strings(requiredTypeKeys) + var requiredTypes []string - for k, c := range table.DbTypes { - if c.IsRequired { + for _, k := range requiredTypeKeys { + if table.DbTypes[k].IsRequired { requiredTypes = append(requiredTypes, k) } } + props.Required = requiredTypes // update properties by their name and type diff --git a/internal/service/handler_common_test.go b/internal/service/handler_common_test.go index 12c57ed0..2395f631 100644 --- a/internal/service/handler_common_test.go +++ b/internal/service/handler_common_test.go @@ -36,24 +36,14 @@ func doRequest(t *testing.T, url string) *httptest.ResponseRecorder { return doRequestStatus(t, url, http.StatusOK) } +func doPostRequest(t *testing.T, url string, data []byte, header http.Header) *httptest.ResponseRecorder { + return doRequestMethodStatus(t, "POST", url, data, header, http.StatusCreated) +} + // do an http request to url with default method GET and a specific expected status func doRequestStatus(t *testing.T, url string, statusExpected int) *httptest.ResponseRecorder { - req, err := http.NewRequest("GET", basePath+url, nil) - if err != nil { - t.Fatal(err) - } - - rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) - - // Check the status code - //fmt.Println("Status:", rr.Code) - if status := rr.Code; status != statusExpected { - t.Errorf("handler returned wrong status code: got %v want %v", - status, statusExpected) - } - return rr + return doRequestMethodStatus(t, "GET", url, nil, nil, statusExpected) } // do an http request to url with a specific method and specific expected status diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index 85111009..0a711d52 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -14,13 +14,17 @@ package service */ import ( + "context" "encoding/json" "fmt" "io/ioutil" "net/http" + "strconv" + "strings" "testing" "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/CrunchyData/pg_featureserv/internal/data" "github.com/getkin/kin-openapi/openapi3" ) @@ -66,3 +70,36 @@ func TestGetCollectionCreateSchema(t *testing.T) { equals(t, "prop_b", fis.Properties["properties"].Value.Required[1], "feature required b") equals(t, "Feature", fis.Properties["type"].Value.Default, "feature required b") } + +func TestCreateFeature(t *testing.T) { + //--- retrieve max feature id + params := data.QueryParam{ + Limit: 100, + Offset: 0, + } + features, _ := catalogMock.TableFeatures(context.Background(), "mock_a", ¶ms) + maxId := len(features) + + var header = make(http.Header) + header.Add("Content-Type", "application/geo+json") + jsonStr := catalogMock.MakeFeatureMockPointAsJSON(maxId, 12, 34) + fmt.Println(jsonStr) + rr := doPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) + + loc := rr.Header().Get("Location") + + assert(t, len(loc) > 1, "Header location must not be empty") + assert(t, strings.Contains(loc, "/collections/mock_a/items/"), "Header location must contain valid data") + + // retrieve new object id from location header + parts := strings.Split(loc, "/") + actId, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + t.Fatal(err) + } + + assert(t, actId > maxId, fmt.Sprintf("Returned id must be > actual feature number: %d > %d", actId, maxId)) + + // check if it can be read + checkItem(t, actId) +} diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index 1edef8e4..4d11af77 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "net/http" "os" + "strconv" "strings" "testing" @@ -165,6 +166,11 @@ func TestCollectionItemsResponse(t *testing.T) { checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") } +// check if item is available and is not empty +func TestCollectionItem(t *testing.T) { + checkItem(t, 1) +} + func TestFilterB(t *testing.T) { rr := doRequest(t, "/collections/mock_a/items?prop_b=1") @@ -499,3 +505,26 @@ func checkFunction(t *testing.T, fun *data.Function) { } checkLink(t, v.Links[2], api.RelItems, itemsType, urlBase+path+"/items") } + +// check if item is available and is not empty +func checkItem(t *testing.T, id int) { + path := fmt.Sprintf("/collections/mock_a/items/%d", id) + resp := doRequest(t, path) + body, _ := ioutil.ReadAll(resp.Body) + + // extracted from catalog_db.go + type featureData struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Geom *json.RawMessage `json:"geometry"` + Props map[string]interface{} `json:"properties"` + } + + var v featureData + json.Unmarshal(body, &v) + + equals(t, "Feature", v.Type, "feature type") + actId, _ := strconv.Atoi(v.ID) + equals(t, id+1, actId, "feature id") // TODO should be 1 ? + equals(t, 4, len(v.Props), "# feature props") +} From 266fbd378b8eb3f85c6bf624225f0a3b1287d4e1 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Wed, 31 Aug 2022 12:21:46 +0200 Subject: [PATCH 05/13] Improve error msg by adding info by using existing Error object. --- internal/service/handler_common_test.go | 13 +++++++++---- internal/service/util.go | 9 +++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/service/handler_common_test.go b/internal/service/handler_common_test.go index 2395f631..4a128f59 100644 --- a/internal/service/handler_common_test.go +++ b/internal/service/handler_common_test.go @@ -71,10 +71,15 @@ func doRequestMethodStatus(t *testing.T, method string, url string, bodyMsg = "Error: " + string(body) } - assert(t, - status == statusExpected, - fmt.Errorf("handler returned wrong status code: got %v want %v. %v", - status, statusExpected, bodyMsg).Error()) + if bodyMsg != "" { + equals(t, + statusExpected, status, + fmt.Errorf("handler returned wrong status code.\n\tCaused by: %v", bodyMsg).Error()) + } else { + equals(t, + statusExpected, status, + "handler returned wrong status code.") + } } return rr } diff --git a/internal/service/util.go b/internal/service/util.go index 37b0f1e5..8c64d3a6 100644 --- a/internal/service/util.go +++ b/internal/service/util.go @@ -91,8 +91,13 @@ func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // log error here? // should log attached error? // panic on severe error? - log.Debugf("Request processing error: %v (%v)\n", e.Message, e.Code) - http.Error(w, e.Message, e.Code) + if e.Error == nil { + log.Debugf("Request processing error: %v (%v)\n", e.Message, e.Code) + http.Error(w, e.Message, e.Code) + } else { + log.Debugf("Request processing error: %v (%v).\n\tCaused by: %v\n", e.Message, e.Code, e.Error) + http.Error(w, fmt.Sprintf("%s\n\tCaused by: %v", e.Message, e.Error), e.Code) + } } close(handlerDone) } From 67fbaa57f2aa9b981a9f1f6d11aa2bb5c1fa74e6 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Wed, 31 Aug 2022 12:22:19 +0200 Subject: [PATCH 06/13] feat(CREATE): add schema validation --- internal/api/api.go | 33 +++++++++------- internal/service/handler.go | 55 +++++++++++++++++++++------ internal/service/handler_post_test.go | 55 +++++++++++++++++---------- internal/service/handler_test.go | 3 +- 4 files changed, 100 insertions(+), 46 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index 36dce0e0..f88f1c88 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -71,21 +71,22 @@ const ( ) const ( - ErrMsgEncoding = "Error encoding response" - ErrMsgLoadCollections = "Unable to access Collections" - ErrMsgCollectionNotFound = "Collection not found: %v" - ErrMsgCollectionAccess = "Unable to access Collection: %v" - ErrMsgFeatureNotFound = "Feature not found: %v" - ErrMsgCreateFeature = "Unable to create new feature in Collection: %v" - ErrMsgLoadFunctions = "Unable to access Functions" - ErrMsgFunctionNotFound = "Function not found: %v" - ErrMsgFunctionAccess = "Unable to access Function: %v" - ErrMsgInvalidParameterValue = "Invalid value for parameter %v: %v" + ErrMsgEncoding = "Error encoding response" + ErrMsgLoadCollections = "Unable to access Collections" + ErrMsgCollectionNotFound = "Collection not found: %v" + ErrMsgCollectionAccess = "Unable to access Collection: %v" + ErrMsgFeatureNotFound = "Feature not found: %v" + ErrMsgCreateFeatureNotConform = "Unable to create new feature in Collection - data does not respect schema: %v" + ErrMsgCreateFeatureInCatalog = "Unable to create new feature in Collection - catalog error: %v" + ErrMsgLoadFunctions = "Unable to access Functions" + ErrMsgFunctionNotFound = "Function not found: %v" + ErrMsgFunctionAccess = "Unable to access Function: %v" + ErrMsgInvalidParameterValue = "Invalid value for parameter %v: %v" ErrMsgInvalidQuery = "Invalid query parameters" - ErrMsgDataReadError = "Unable to read data from: %v" + ErrMsgDataReadError = "Unable to read data from: %v" ErrMsgDataWriteError = "Unable to write data to: %v" - ErrMsgNoDataRead = "No data read from: %v" - ErrMsgRequestTimeout = "Maximum time exceeded. Request cancelled." + ErrMsgNoDataRead = "No data read from: %v" + ErrMsgRequestTimeout = "Maximum time exceeded. Request cancelled." ) const ( @@ -641,3 +642,9 @@ func PathFunctionItems(name string) string { func PathItem(name string, fid string) string { return fmt.Sprintf("%v/%v/%v/%v", TagCollections, name, TagItems, fid) } + +var Db2OpenapiFormatMap = map[string]string{ + "int": "integer", + "long": "int64", + "text": "string", +} diff --git a/internal/service/handler.go b/internal/service/handler.go index 29357a9b..8ee30a5d 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -16,6 +16,7 @@ package service import ( "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -267,7 +268,6 @@ func handleCollection(w http.ResponseWriter, r *http.Request) *appError { func handleCollectionSchemas(w http.ResponseWriter, r *http.Request) *appError { // TODO: determine content from request header? format := api.RequestedFormat(r) - urlBase := serveURLBase(r) //--- extract request parameters name := getRequestVar(routeVarID, r) @@ -290,7 +290,7 @@ func handleCollectionSchemas(w http.ResponseWriter, r *http.Request) *appError { { switch schemaType { case "create": - return writeCreateItemSchemaJSON(ctx, w, tbl, urlBase) + return writeCreateItemSchemaJSON(ctx, w, tbl) default: return appErrorBadRequest(nil, fmt.Sprintf("Asked schema type %s not implemented!", schemaType)) } @@ -326,19 +326,30 @@ func handleCreateCollectionItem(w http.ResponseWriter, r *http.Request) *appErro } //--- json body - body, errBody := ioutil.ReadAll(r.Body) - if errBody != nil || len(body) == 0 { + bodyContent, errBody := ioutil.ReadAll(r.Body) + if errBody != nil || len(bodyContent) == 0 { return appErrorInternalFmt(errBody, "Unable to read request body for Collection: %v", name) } - newId, err2 := catalogInstance.AddTableFeature(r.Context(), name, body) + //--- check if body matches the schema + createSchema, errGetSch := getCreateItemSchema(r.Context(), tbl) + if errGetSch != nil { + return appErrorInternalFmt(errGetSch, errGetSch.Error()) + } + var val interface{} + _ = json.Unmarshal(bodyContent, &val) + errValSch := createSchema.VisitJSON(val) + if errValSch != nil { + return appErrorInternalFmt(errValSch, api.ErrMsgCreateFeatureNotConform, name) + } + + newId, err2 := catalogInstance.AddTableFeature(r.Context(), name, bodyContent) if err2 != nil { - return appErrorInternalFmt(err2, api.ErrMsgCreateFeature, name) + return appErrorInternalFmt(err2, api.ErrMsgCreateFeatureInCatalog, name) } w.Header().Set("Location", fmt.Sprintf("%scollections/%s/items/%d", urlBase, name, newId)) w.WriteHeader(http.StatusCreated) - w.Write([]byte("")) return nil } @@ -378,13 +389,22 @@ func handleCollectionItems(w http.ResponseWriter, r *http.Request) *appError { return nil } -func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table *data.Table, urlBase string) *appError { +func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table *data.Table) *appError { + createSchema, err := getCreateItemSchema(ctx, table) + if err != nil { + return appErrorMsg(err, err.Error(), http.StatusInternalServerError) + } + return writeJSON(w, api.ContentTypeSchemaJSON, createSchema) + +} + +func getCreateItemSchema(ctx context.Context, table *data.Table) (openapi3.Schema, error) { // Feature schema skeleton var featureInfoSchema openapi3.Schema = openapi3.Schema{ Type: "object", Required: []string{"type", "geometry", "properties"}, Properties: map[string]*openapi3.SchemaRef{ - "id": {Value: &openapi3.Schema{Type: "number", Format: "long"}}, + "id": {Value: &openapi3.Schema{Type: "string", Format: "uri"}}, "type": { Value: &openapi3.Schema{ Type: "string", @@ -395,6 +415,9 @@ func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table Value: &openapi3.Schema{ Items: &openapi3.SchemaRef{ Ref: fmt.Sprintf("https://geojson.org/schema/%v.json", table.GeometryType), + Value: &openapi3.Schema{ + Type: "string", // mandatory to validate the schema + }, }, }, }, @@ -428,14 +451,24 @@ func writeCreateItemSchemaJSON(ctx context.Context, w http.ResponseWriter, table // update properties by their name and type props.Properties = make(map[string]*openapi3.SchemaRef) for k, v := range table.DbTypes { + propType := v.Type + if api.Db2OpenapiFormatMap[v.Type] != "" { + propType = api.Db2OpenapiFormatMap[v.Type] + } props.Properties[k] = &openapi3.SchemaRef{ Value: &openapi3.Schema{ - Type: v.Type, + Type: propType, }, } } - return writeJSON(w, api.ContentTypeSchemaJSON, featureInfoSchema) + errVal := featureInfoSchema.Validate(ctx) + if errVal != nil { + encodedContent, _ := json.Marshal(featureInfoSchema) + return featureInfoSchema, fmt.Errorf("schema not valid: %v\n\t%v", errVal, string(encodedContent)) + } + + return featureInfoSchema, nil } func writeItemsHTML(w http.ResponseWriter, tbl *data.Table, name string, query string, urlBase string) *appError { diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index 0a711d52..105a640a 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -56,13 +56,9 @@ func TestGetCollectionCreateSchema(t *testing.T) { resp := doRequestMethodStatus(t, "GET", path, nil, header, http.StatusOK) body, _ := ioutil.ReadAll(resp.Body) - fmt.Println(string(body)) - var fis openapi3.Schema - err := fis.UnmarshalJSON(body) - if err != nil { - t.Fatal(err) - } + errUnMarsh := json.Unmarshal(body, &fis) + assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) equals(t, "This dataset contains mock data about A (9 points)", fis.Description, "feature description") equals(t, "https://geojson.org/schema/Point.json", fis.Properties["geometry"].Value.Items.Ref, "feature geometry") @@ -82,24 +78,41 @@ func TestCreateFeature(t *testing.T) { var header = make(http.Header) header.Add("Content-Type", "application/geo+json") - jsonStr := catalogMock.MakeFeatureMockPointAsJSON(maxId, 12, 34) - fmt.Println(jsonStr) - rr := doPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) + { + jsonStr := `[{ + "id": 101, + "name": "Test", + "email": "test@test.com" + }, { + "id": 102, + "name": "Sample", + "email": "sample@test.com" + }]` + rr := doRequestMethodStatus(t, "POST", "/collections/mock_a/items", []byte(jsonStr), header, http.StatusInternalServerError) + equals(t, http.StatusInternalServerError, rr.Code, "Should have failed") + assert(t, strings.Index(rr.Body.String(), fmt.Sprintf(api.ErrMsgCreateFeatureNotConform+"\n", "mock_a")) == 0, "Should have failed with not conform") + } - loc := rr.Header().Get("Location") + { + jsonStr := catalogMock.MakeFeatureMockPointAsJSON(maxId, 12, 34) + fmt.Println(jsonStr) + rr := doPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) - assert(t, len(loc) > 1, "Header location must not be empty") - assert(t, strings.Contains(loc, "/collections/mock_a/items/"), "Header location must contain valid data") + loc := rr.Header().Get("Location") - // retrieve new object id from location header - parts := strings.Split(loc, "/") - actId, err := strconv.Atoi(parts[len(parts)-1]) - if err != nil { - t.Fatal(err) - } + assert(t, len(loc) > 1, "Header location must not be empty") + assert(t, strings.Contains(loc, "/collections/mock_a/items/"), "Header location must contain valid data") + + // retrieve new object id from location header + parts := strings.Split(loc, "/") + actId, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + t.Fatal(err) + } - assert(t, actId > maxId, fmt.Sprintf("Returned id must be > actual feature number: %d > %d", actId, maxId)) + assert(t, actId > maxId, fmt.Sprintf("Returned id must be > actual feature number: %d > %d", actId, maxId)) - // check if it can be read - checkItem(t, actId) + // check if it can be read + checkItem(t, actId) + } } diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index 4d11af77..af00f2c3 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -521,7 +521,8 @@ func checkItem(t *testing.T, id int) { } var v featureData - json.Unmarshal(body, &v) + errUnMarsh := json.Unmarshal(body, &v) + assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) equals(t, "Feature", v.Type, "feature type") actId, _ := strconv.Atoi(v.ID) From a3d922faed63e072973062be5a881bbae2784f9f Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Thu, 1 Sep 2022 13:41:25 +0200 Subject: [PATCH 07/13] Add geojson geometry management in catalog --- go.mod | 1 + go.sum | 53 +++++++++++++++++++++++++++-------- internal/data/catalog_db.go | 50 ++++++++++++++++++++++++++------- internal/data/catalog_mock.go | 21 +++++++------- 4 files changed, 94 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 4e19d79f..7a28a069 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/gorilla/mux v1.7.3 github.com/jackc/pgtype v1.0.2 github.com/jackc/pgx/v4 v4.1.2 + github.com/paulmach/orb v0.7.1 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 github.com/sirupsen/logrus v1.7.0 github.com/spf13/viper v1.6.1 diff --git a/go.sum b/go.sum index 32510834..8bda7912 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antlr/antlr4 v0.0.0-20220209173558-ad29539cd2e9 h1:NAjo5dlbySnIEFVRSgPwy+WjEDSY4EL6baGKvvPI/QM= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9 h1:zvkJv+9Pxm1nnEMcKnShREt4qtduHKz4iw4AB4ul0Ao= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -30,26 +29,27 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.2.0 h1:PbHHtYZpjKwZtGlIyELgA2DploRrsaXztoNNx9HjwNY= github.com/getkin/kin-openapi v0.2.0/go.mod h1:V1z9xl9oF5Wt7v32ne4FmiF1alpS4dM6mNzoywPOXlk= -github.com/getkin/kin-openapi v0.3.0 h1:xsQ4mA20YJDMgIHdHqMKZ66QUe6/hi+x6yLsTTz8xyQ= -github.com/getkin/kin-openapi v0.32.0 h1:zzeKoKewdKT9bBdRMwYEhJgB2rKSKGFKE1iOKF7KgNs= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-spatial/jivan v0.1.3 h1:UmyYh06iTECHqG+OZ5Vktn1pavEoRj5rBqGFjKy0XyY= -github.com/go-spatial/jivan v0.1.3/go.mod h1:BvRLfq/DOULog8GRavvcifGydUAGP1UrZD7ZO3l9Oi4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -72,6 +72,7 @@ github.com/jackc/pgconn v1.1.0 h1:10i6DMVJOSko/sD3FLpFKBHONzDGKkX8pbLyHC8B92o= github.com/jackc/pgconn v1.1.0/go.mod h1:GgY/Lbj1VonNaVdNUHs9AwWom3yP2eymFQ1C8z9r/Lk= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -98,19 +99,24 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.0.0 h1:rbjAshlgKscNa7j0jAM0uNQflis5o2XUogPMVAwtcsM= github.com/jackc/puddle v1.0.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -125,6 +131,9 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/paulmach/orb v0.7.1 h1:Zha++Z5OX/l168sqHK3k4z18LDvr+YAO/VjK0ReQ9rU= +github.com/paulmach/orb v0.7.1/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -148,14 +157,16 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -184,6 +195,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -195,10 +208,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -207,11 +224,15 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -222,13 +243,14 @@ golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -237,16 +259,25 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= diff --git a/internal/data/catalog_db.go b/internal/data/catalog_db.go index e015c113..fadacbb8 100644 --- a/internal/data/catalog_db.go +++ b/internal/data/catalog_db.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "sort" "strings" "time" @@ -26,6 +27,7 @@ import ( "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/log/logrusadapter" "github.com/jackc/pgx/v4/pgxpool" + "github.com/paulmach/orb/geojson" log "github.com/sirupsen/logrus" ) @@ -432,29 +434,33 @@ func scanFeatures(ctx context.Context, rows pgx.Rows, idColIndex int, propCols [ } func scanFeature(rows pgx.Rows, idColIndex int, propNames []string) string { - var id, geom string + var id string + vals, err := rows.Values() if err != nil { log.Warnf("Error scanning row for Feature: %v", err) return "" } //fmt.Println(vals) - //--- geom value is expected to be a GeoJSON string - //--- convert NULL to an empty string - if vals[0] != nil { - geom = vals[0].(string) - } else { - geom = "" - } propOffset := 1 if idColIndex >= 0 { id = fmt.Sprintf("%v", vals[idColIndex+propOffset]) } - //fmt.Println(geom) props := extractProperties(vals, propOffset, propNames) - return makeFeatureJSON(id, geom, props) + + //--- geom value is expected to be a GeoJSON string or geojson object + //--- convert NULL to an empty string + if vals[0] != nil { + if "string" == reflect.TypeOf(vals[0]).String() { + return makeFeatureJSON(id, vals[0].(string), props) + } else { + return makeGeojsonFeatureJSON(id, vals[0].(geojson.Geometry), props) + } + } else { + return makeFeatureJSON(id, "", props) + } } func extractProperties(vals []interface{}, propOffset int, propNames []string) map[string]interface{} { @@ -588,6 +594,30 @@ func makeFeatureJSON(id string, geom string, props map[string]interface{}) strin return jsonStr } +type geojsonFeatureData struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Geom *geojson.Geometry `json:"geometry"` + Props map[string]interface{} `json:"properties"` +} + +func makeGeojsonFeatureJSON(id string, geom geojson.Geometry, props map[string]interface{}) string { + featData := geojsonFeatureData{ + Type: "Feature", + ID: id, + Geom: &geom, + Props: props, + } + json, err := json.Marshal(featData) + if err != nil { + log.Errorf("Error marshalling feature into JSON: %v", err) + return "" + } + jsonStr := string(json) + //fmt.Println(jsonStr) + return jsonStr +} + // indexOfName finds the index of a name in an array of names // It returns the index or -1 if not found func indexOfName(names []string, name string) int { diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index 5b987296..4d62966e 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -18,6 +18,8 @@ import ( "fmt" "strconv" + orb "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" log "github.com/sirupsen/logrus" ) @@ -303,26 +305,25 @@ func makePointFeatures(extent Extent, nx int, ny int) []*featureMock { } type featureMock struct { - ID string - Geom string - PropA string - PropB int - PropC string - PropD int + ID string `json:"ID"` + Geom *geojson.Geometry `json:"geometry"` + PropA string `json:"prop_a"` + PropB int `json:"prop_b"` + PropC string `json:"prop_c"` + PropD int `json:"prop_d"` } func makeFeatureMockPoint(id int, x float64, y float64) *featureMock { - geomFmt := `{"type": "Point","coordinates": [ %v, %v ] }` - geomStr := fmt.Sprintf(geomFmt, x, y) + geom := geojson.NewGeometry(orb.Point{x, y}) idstr := strconv.Itoa(id) - feat := featureMock{idstr, geomStr, "propA", id, "propC", id % 10} + feat := featureMock{idstr, geom, "propA", id, "propC", id % 10} return &feat } func (fm *featureMock) toJSON(propNames []string) string { props := fm.extractProperties(propNames) - return makeFeatureJSON(fm.ID, fm.Geom, props) + return makeGeojsonFeatureJSON(fm.ID, *fm.Geom, props) } func (fm *featureMock) extractProperties(propNames []string) map[string]interface{} { From e386435d4ff66fd332b1300746b470a802145ab0 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Thu, 1 Sep 2022 13:42:19 +0200 Subject: [PATCH 08/13] feat(CREATE): implements AddTableFeature in catalog_mock --- internal/data/catalog_mock.go | 28 ++++++++++++++++++++++++++- internal/service/handler_post_test.go | 28 +++++++-------------------- internal/service/handler_test.go | 10 +++++----- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index 4d62966e..c7f7e81b 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -15,6 +15,7 @@ package data import ( "context" + "encoding/json" "fmt" "strconv" @@ -234,6 +235,8 @@ func (cat *CatalogMock) TableFeature(ctx context.Context, name string, id string return "", nil } + index-- + // TODO: return not found if index out of range if index < 0 || index >= len(features) { return "", nil @@ -247,8 +250,31 @@ func (cat *CatalogMock) TableFeature(ctx context.Context, name string, id string return features[index].toJSON(propNames), nil } +// returns the number of feature for a specific table +func (cat *CatalogMock) TableSize(tableName string) int64 { + return int64(len(cat.tableData[tableName])) +} + func (cat *CatalogMock) AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) { - panic("CatalogMock::AddTableFeature unimplemented") + var newFeature featureMock + + var schemaObject geojsonFeatureData + err := json.Unmarshal(jsonData, &schemaObject) + if err != nil { + return 0, err + } + + maxId := cat.TableSize(tableName) + + newFeature.ID = fmt.Sprintf("%d", maxId+1) + newFeature.Geom = schemaObject.Geom + newFeature.PropA = schemaObject.Props["prop_a"].(string) + newFeature.PropB = int(schemaObject.Props["prop_b"].(float64)) + newFeature.PropC = schemaObject.Props["prop_c"].(string) + newFeature.PropD = int(schemaObject.Props["prop_d"].(float64)) + + cat.tableData[tableName] = append(cat.tableData[tableName], &newFeature) + return maxId + 1, nil } func (cat *CatalogMock) Functions() ([]*Function, error) { diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index 105a640a..b3bc0700 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strconv" "strings" "testing" @@ -68,14 +67,6 @@ func TestGetCollectionCreateSchema(t *testing.T) { } func TestCreateFeature(t *testing.T) { - //--- retrieve max feature id - params := data.QueryParam{ - Limit: 100, - Offset: 0, - } - features, _ := catalogMock.TableFeatures(context.Background(), "mock_a", ¶ms) - maxId := len(features) - var header = make(http.Header) header.Add("Content-Type", "application/geo+json") { @@ -94,25 +85,20 @@ func TestCreateFeature(t *testing.T) { } { - jsonStr := catalogMock.MakeFeatureMockPointAsJSON(maxId, 12, 34) + jsonStr := catalogMock.MakeFeatureMockPointAsJSON(0, 12, 34) fmt.Println(jsonStr) rr := doPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) loc := rr.Header().Get("Location") + //--- retrieve max feature id + params := data.QueryParam{Limit: 100, Offset: 0} + features, _ := catalogMock.TableFeatures(context.Background(), "mock_a", ¶ms) + maxId := len(features) assert(t, len(loc) > 1, "Header location must not be empty") - assert(t, strings.Contains(loc, "/collections/mock_a/items/"), "Header location must contain valid data") - - // retrieve new object id from location header - parts := strings.Split(loc, "/") - actId, err := strconv.Atoi(parts[len(parts)-1]) - if err != nil { - t.Fatal(err) - } - - assert(t, actId > maxId, fmt.Sprintf("Returned id must be > actual feature number: %d > %d", actId, maxId)) + assert(t, strings.Contains(loc, fmt.Sprintf("/collections/mock_a/items/%d", maxId)), "Header location must contain valid data") // check if it can be read - checkItem(t, actId) + checkItem(t, maxId) } } diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index af00f2c3..74ca6c17 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -161,7 +161,7 @@ func TestCollectionItemsResponse(t *testing.T) { errUnMarsh := json.Unmarshal(body, &v) assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 9, len(v.Features), "# features") + equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") } @@ -218,7 +218,7 @@ func TestSortBy(t *testing.T) { errUnMarsh := json.Unmarshal(readBody(rr), &v) assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 9, len(v.Features), "# features") + equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestSortByDesc(t *testing.T) { @@ -228,7 +228,7 @@ func TestSortByDesc(t *testing.T) { errUnMarsh := json.Unmarshal(readBody(rr), &v) assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 9, len(v.Features), "# features") + equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestSortByAsc(t *testing.T) { @@ -238,7 +238,7 @@ func TestSortByAsc(t *testing.T) { errUnMarsh := json.Unmarshal(readBody(rr), &v) assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 9, len(v.Features), "# features") + equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestLimit(t *testing.T) { @@ -526,6 +526,6 @@ func checkItem(t *testing.T, id int) { equals(t, "Feature", v.Type, "feature type") actId, _ := strconv.Atoi(v.ID) - equals(t, id+1, actId, "feature id") // TODO should be 1 ? + equals(t, id, actId, "feature id") equals(t, 4, len(v.Props), "# feature props") } From a8248b970058fddb145a7e3c4875e6aaaf788cb2 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Fri, 2 Sep 2022 15:21:00 +0200 Subject: [PATCH 09/13] test: create new package util with reworked test tools (http, assert/equals, ...) --- internal/cql/cql_test.go | 29 +- internal/data/catalog_mock.go | 5 +- internal/service/empty_base_path_test.go | 23 +- internal/service/handler.go | 2 +- internal/service/handler_post_test.go | 54 +-- internal/service/handler_test.go | 351 ++++++++---------- internal/service/service.go | 9 +- util/test_common.go | 51 +++ .../test_http.go | 92 +++-- 9 files changed, 325 insertions(+), 291 deletions(-) create mode 100644 util/test_common.go rename internal/service/handler_common_test.go => util/test_http.go (51%) diff --git a/internal/cql/cql_test.go b/internal/cql/cql_test.go index 90c01b09..c543ec79 100644 --- a/internal/cql/cql_test.go +++ b/internal/cql/cql_test.go @@ -15,11 +15,10 @@ package cql import ( "fmt" - "path/filepath" - "reflect" - "runtime" "strings" "testing" + + "github.com/CrunchyData/pg_featureserv/util" ) func TestDebug(t *testing.T) { @@ -169,7 +168,7 @@ func checkCQL(t *testing.T, cqlStr string, sql string) { t.FailNow() } actual = strings.TrimSpace(actual) - equals(t, sql, actual, "") + util.Equals(t, sql, actual, "") } func checkCQLWithSRID(t *testing.T, cqlStr string, filterSRID int, sourceSRID int, sql string) { @@ -179,28 +178,10 @@ func checkCQLWithSRID(t *testing.T, cqlStr string, filterSRID int, sourceSRID in t.FailNow() } actual = strings.TrimSpace(actual) - equals(t, sql, actual, "") + util.Equals(t, sql, actual, "") } func checkCQLError(t *testing.T, cqlStr string) { _, err := TranspileToSQL(cqlStr, 4326, 4326) - isError(t, err, "") -} - -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}, msg string) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: %s - expected: %#v; got: %#v\n", filepath.Base(file), line, msg, exp, act) - tb.FailNow() - } -} - -// isError fails the test if err is nil -func isError(tb testing.TB, err error, msg string) { - if err == nil { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: %s - expected error\n", filepath.Base(file), line, msg) - tb.FailNow() - } + util.AssertIsError(t, err, "") } diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index c7f7e81b..04872ffc 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -301,10 +301,9 @@ func (cat *CatalogMock) FunctionData(ctx context.Context, name string, args map[ return nil, nil } -func (cat *CatalogMock) MakeFeatureMockPointAsJSON(id int, x float64, y float64) string { +func MakeFeatureMockPointAsJSON(id int, x float64, y float64, columns []string) string { feat := makeFeatureMockPoint(id, x, y) - propNames := cat.TableDefs[0].Columns - return feat.toJSON(propNames) + return feat.toJSON(columns) } func makePointFeatures(extent Extent, nx int, ny int) []*featureMock { diff --git a/internal/service/empty_base_path_test.go b/internal/service/empty_base_path_test.go index 55063849..53b13357 100644 --- a/internal/service/empty_base_path_test.go +++ b/internal/service/empty_base_path_test.go @@ -16,26 +16,23 @@ package service import ( "fmt" "testing" + + "github.com/CrunchyData/pg_featureserv/util" ) func TestRootEmptyBasePath(t *testing.T) { - basePath = "" - setup(basePath) + hTestBadPath := util.MakeHttpTesting("http://test", "", InitRouter("")) Initialize() testCases := []string{ - "/", + "/", "/index.html", - - } + } for _, tc := range testCases { - t.Run(fmt.Sprintf("%s route works with empty base path", tc), func(t *testing.T) { - resp := doRequest(t, tc) - assert(t, resp.Code == 200, "Status must be 200") - }) - } + t.Run(fmt.Sprintf("%s route works with empty base path", tc), func(t *testing.T) { + resp := hTestBadPath.DoRequest(t, tc) + util.Assert(t, resp.Code == 200, "Status must be 200") + }) + } - basePath = "/pg_featureserv" - setup(basePath) - Initialize() } diff --git a/internal/service/handler.go b/internal/service/handler.go index 8ee30a5d..478bdd2a 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -36,7 +36,7 @@ const ( routeVarFeatureID = "fid" ) -func initRouter(basePath string) *mux.Router { +func InitRouter(basePath string) *mux.Router { router := mux.NewRouter(). StrictSlash(true). PathPrefix("/" + strings.TrimRight(strings.TrimLeft(basePath, "/"), "/")). diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index b3bc0700..64c41a33 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -24,26 +24,27 @@ import ( "github.com/CrunchyData/pg_featureserv/internal/api" "github.com/CrunchyData/pg_featureserv/internal/data" + "github.com/CrunchyData/pg_featureserv/util" "github.com/getkin/kin-openapi/openapi3" ) // checks swagger api contains get operation from collection schema func TestApiContainsCollectionSchemas(t *testing.T) { - resp := doRequest(t, "/api") + resp := hTest.DoRequest(t, "/api") body, _ := ioutil.ReadAll(resp.Body) var v openapi3.Swagger errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 11, len(v.Paths), "# api paths") + util.Equals(t, 11, len(v.Paths), "# api paths") path := v.Paths.Find("/collections/{collectionId}/schema") - assert(t, path != nil, "schema path exists") - equals(t, "Provides access to data representation (schema) for any features in specified collection", path.Description, "schema path present") - equals(t, "getCollectionSchema", path.Get.OperationID, "schema path get present") - equals(t, 2, len(path.Get.Parameters), "schema path get present") - assert(t, path.Get.Parameters.GetByInAndName("path", "collectionId") != nil, "collectionId path parameter exists") - assert(t, path.Get.Parameters.GetByInAndName("query", "type") != nil, "type query parameter exists") + util.Assert(t, path != nil, "schema path exists") + util.Equals(t, "Provides access to data representation (schema) for any features in specified collection", path.Description, "schema path present") + util.Equals(t, "getCollectionSchema", path.Get.OperationID, "schema path get present") + util.Equals(t, 2, len(path.Get.Parameters), "schema path get present") + util.Assert(t, path.Get.Parameters.GetByInAndName("path", "collectionId") != nil, "collectionId path parameter exists") + util.Assert(t, path.Get.Parameters.GetByInAndName("query", "type") != nil, "type query parameter exists") } // checks collection schema contains valid data description @@ -52,18 +53,18 @@ func TestGetCollectionCreateSchema(t *testing.T) { var header = make(http.Header) header.Add("Accept", api.ContentTypeSchemaJSON) - resp := doRequestMethodStatus(t, "GET", path, nil, header, http.StatusOK) + resp := hTest.DoRequestMethodStatus(t, "GET", path, nil, header, http.StatusOK) body, _ := ioutil.ReadAll(resp.Body) var fis openapi3.Schema errUnMarsh := json.Unmarshal(body, &fis) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, "This dataset contains mock data about A (9 points)", fis.Description, "feature description") - equals(t, "https://geojson.org/schema/Point.json", fis.Properties["geometry"].Value.Items.Ref, "feature geometry") - equals(t, "prop_a", fis.Properties["properties"].Value.Required[0], "feature required a") - equals(t, "prop_b", fis.Properties["properties"].Value.Required[1], "feature required b") - equals(t, "Feature", fis.Properties["type"].Value.Default, "feature required b") + util.Equals(t, "This dataset contains mock data about A (9 points)", fis.Description, "feature description") + util.Equals(t, "https://geojson.org/schema/Point.json", fis.Properties["geometry"].Value.Items.Ref, "feature geometry") + util.Equals(t, "prop_a", fis.Properties["properties"].Value.Required[0], "feature required a") + util.Equals(t, "prop_b", fis.Properties["properties"].Value.Required[1], "feature required b") + util.Equals(t, "Feature", fis.Properties["type"].Value.Default, "feature required b") } func TestCreateFeature(t *testing.T) { @@ -79,15 +80,22 @@ func TestCreateFeature(t *testing.T) { "name": "Sample", "email": "sample@test.com" }]` - rr := doRequestMethodStatus(t, "POST", "/collections/mock_a/items", []byte(jsonStr), header, http.StatusInternalServerError) - equals(t, http.StatusInternalServerError, rr.Code, "Should have failed") - assert(t, strings.Index(rr.Body.String(), fmt.Sprintf(api.ErrMsgCreateFeatureNotConform+"\n", "mock_a")) == 0, "Should have failed with not conform") + rr := hTest.DoRequestMethodStatus(t, "POST", "/collections/mock_a/items", []byte(jsonStr), header, http.StatusInternalServerError) + util.Equals(t, http.StatusInternalServerError, rr.Code, "Should have failed") + util.Assert(t, strings.Index(rr.Body.String(), fmt.Sprintf(api.ErrMsgCreateFeatureNotConform+"\n", "mock_a")) == 0, "Should have failed with not conform") } { - jsonStr := catalogMock.MakeFeatureMockPointAsJSON(0, 12, 34) + var cols []string + for _, t := range catalogMock.TableDefs { + if t.ID == "mock_a" { + cols = t.Columns + break + } + } + jsonStr := data.MakeFeatureMockPointAsJSON(0, 12, 34, cols) fmt.Println(jsonStr) - rr := doPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) + rr := hTest.DoPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) loc := rr.Header().Get("Location") @@ -95,8 +103,8 @@ func TestCreateFeature(t *testing.T) { params := data.QueryParam{Limit: 100, Offset: 0} features, _ := catalogMock.TableFeatures(context.Background(), "mock_a", ¶ms) maxId := len(features) - assert(t, len(loc) > 1, "Header location must not be empty") - assert(t, strings.Contains(loc, fmt.Sprintf("/collections/mock_a/items/%d", maxId)), "Header location must contain valid data") + util.Assert(t, len(loc) > 1, "Header location must not be empty") + util.Assert(t, strings.Contains(loc, fmt.Sprintf("/collections/mock_a/items/%d", maxId)), "Header location must contain valid data") // check if it can be read checkItem(t, maxId) diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index 74ca6c17..86cfddfb 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -24,8 +24,8 @@ import ( "testing" "github.com/CrunchyData/pg_featureserv/internal/api" - "github.com/CrunchyData/pg_featureserv/internal/conf" "github.com/CrunchyData/pg_featureserv/internal/data" + "github.com/CrunchyData/pg_featureserv/util" ) // Define a FeatureCollection structure for parsing test data @@ -46,59 +46,34 @@ type FeatureCollection struct { Links []*api.Link `json:"links"` } -const urlBase = "http://test" - -var basePath = "/pg_featureserv" +var hTest util.HttpTesting var catalogMock *data.CatalogMock func TestMain(m *testing.M) { catalogMock = data.CatMockInstance() catalogInstance = catalogMock - setup(basePath) + + hTest = util.MakeHttpTesting("http://test", "/pg_featureserv", InitRouter("/pg_featureserv")) Initialize() - os.Exit(m.Run()) -} -func setup(path string) { - router = initRouter(path) - conf.Configuration = conf.Config{ - Server: conf.Server{ - HttpHost: "0.0.0.0", - HttpPort: 9000, - UrlBase: urlBase, - BasePath: path, - AssetsPath: "../../assets", - TransformFunctions: []string{ - "ST_Centroid", - "ST_PointOnSurface", - }, - }, - Paging: conf.Paging{ - LimitDefault: 10, - LimitMax: 1000, - }, - Metadata: conf.Metadata{ - Title: "test", - Description: "test", - }, - } + os.Exit(m.Run()) } func TestRoot(t *testing.T) { - resp := doRequest(t, "/") + resp := hTest.DoRequest(t, "/") body, _ := ioutil.ReadAll(resp.Body) var v api.RootInfo errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+"/"+api.RootPageName) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+"/"+api.RootPageName+".html") - checkLink(t, v.Links[2], api.RelServiceDesc, api.ContentTypeOpenAPI, urlBase+"/api") - checkLink(t, v.Links[3], api.RelConformance, api.ContentTypeJSON, urlBase+"/conformance") - checkLink(t, v.Links[4], api.RelData, api.ContentTypeJSON, urlBase+"/collections") - checkLink(t, v.Links[5], api.RelFunctions, api.ContentTypeJSON, urlBase+"/functions") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+"/"+api.RootPageName) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+"/"+api.RootPageName+".html") + checkLink(t, v.Links[2], api.RelServiceDesc, api.ContentTypeOpenAPI, hTest.UrlBase+"/api") + checkLink(t, v.Links[3], api.RelConformance, api.ContentTypeJSON, hTest.UrlBase+"/conformance") + checkLink(t, v.Links[4], api.RelData, api.ContentTypeJSON, hTest.UrlBase+"/collections") + checkLink(t, v.Links[5], api.RelFunctions, api.ContentTypeJSON, hTest.UrlBase+"/functions") /* fmt.Println("Response ==>") @@ -109,15 +84,15 @@ func TestRoot(t *testing.T) { func TestCollectionsResponse(t *testing.T) { path := "/collections" - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) var v api.CollectionsInfo errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") checkCollection(t, v.Collections[0], "mock_a", "Mock A") checkCollection(t, v.Collections[1], "mock_b", "Mock B") @@ -126,44 +101,44 @@ func TestCollectionsResponse(t *testing.T) { func TestCollectionResponse(t *testing.T) { path := "/collections/mock_a" - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) var v api.CollectionInfo errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) // use mock data as expected tbl := catalogMock.TableDefs[0] - equals(t, tbl.ID, v.Name, "Name") - equals(t, tbl.Title, v.Title, "Title") - equals(t, tbl.Description, v.Description, "Description") + util.Equals(t, tbl.ID, v.Name, "Name") + util.Equals(t, tbl.Title, v.Title, "Title") + util.Equals(t, tbl.Description, v.Description, "Description") // check properties - equals(t, len(tbl.Columns), len(v.Properties), "Properties len") + util.Equals(t, len(tbl.Columns), len(v.Properties), "Properties len") for i := 0; i < len(v.Properties); i++ { - equals(t, tbl.Columns[i], v.Properties[i].Name, "Properties[].Name") - equals(t, tbl.JSONTypes[i], v.Properties[i].Type, "Properties[].Type") - equals(t, tbl.ColDesc[i], v.Properties[i].Description, "Properties[].Description") + util.Equals(t, tbl.Columns[i], v.Properties[i].Name, "Properties[].Name") + util.Equals(t, tbl.JSONTypes[i], v.Properties[i].Type, "Properties[].Type") + util.Equals(t, tbl.ColDesc[i], v.Properties[i].Description, "Properties[].Description") } - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") - checkLink(t, v.Links[2], api.RelItems, api.ContentTypeGeoJSON, urlBase+path+"/items") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") + checkLink(t, v.Links[2], api.RelItems, api.ContentTypeGeoJSON, hTest.UrlBase+path+"/items") } func TestCollectionItemsResponse(t *testing.T) { path := "/collections/mock_a/items" - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) var v api.FeatureCollectionRaw errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + util.Equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") } // check if item is available and is not empty @@ -172,213 +147,213 @@ func TestCollectionItem(t *testing.T) { } func TestFilterB(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?prop_b=1") + rr := hTest.DoRequest(t, "/collections/mock_a/items?prop_b=1") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 1, len(v.Features), "# features") + util.Equals(t, 1, len(v.Features), "# features") } func TestFilterD(t *testing.T) { - rr := doRequest(t, "/collections/mock_c/items?prop_d=1") + rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_d=1") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 10, len(v.Features), "# features") + util.Equals(t, 10, len(v.Features), "# features") } func TestFilterBD(t *testing.T) { - rr := doRequest(t, "/collections/mock_c/items?prop_b=2&prop_d=2") + rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_b=2&prop_d=2") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 1, len(v.Features), "# features") + util.Equals(t, 1, len(v.Features), "# features") } func TestFilterBDNone(t *testing.T) { - rr := doRequest(t, "/collections/mock_c/items?prop_b=1&prop_d=2") + rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_b=1&prop_d=2") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 0, len(v.Features), "# features") + util.Equals(t, 0, len(v.Features), "# features") } func TestSortBy(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?sortby=prop_b") + rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=prop_b") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") + util.Equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestSortByDesc(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?sortby=-prop_b") + rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=-prop_b") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") + util.Equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestSortByAsc(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?sortby=+prop_b") + rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=+prop_b") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") + util.Equals(t, int(catalogMock.TableSize("mock_a")), len(v.Features), "# features") } func TestLimit(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?limit=3") + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=3") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 3, len(v.Features), "# features") - equals(t, "1", v.Features[0].ID, "feature 1 id") - equals(t, "2", v.Features[1].ID, "feature 2 id") - equals(t, "3", v.Features[2].ID, "feature 3 id") + util.Equals(t, 3, len(v.Features), "# features") + util.Equals(t, "1", v.Features[0].ID, "feature 1 id") + util.Equals(t, "2", v.Features[1].ID, "feature 2 id") + util.Equals(t, "3", v.Features[2].ID, "feature 3 id") } func TestLimitZero(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?limit=0") + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=0") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, "FeatureCollection", v.Type, "type FeatureCollection") - equals(t, 0, len(v.Features), "# features") + util.Equals(t, "FeatureCollection", v.Type, "type FeatureCollection") + util.Equals(t, 0, len(v.Features), "# features") } func TestLimitInvalid(t *testing.T) { - doRequestStatus(t, "/collections/mock_a/items?limit=x", http.StatusBadRequest) + hTest.DoRequestStatus(t, "/collections/mock_a/items?limit=x", http.StatusBadRequest) } func TestQueryParamCase(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?LIMIT=2&Offset=4") + rr := hTest.DoRequest(t, "/collections/mock_a/items?LIMIT=2&Offset=4") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 2, len(v.Features), "# features") - equals(t, "5", v.Features[0].ID, "feature 5 id") - equals(t, "6", v.Features[1].ID, "feature 6 id") + util.Equals(t, 2, len(v.Features), "# features") + util.Equals(t, "5", v.Features[0].ID, "feature 5 id") + util.Equals(t, "6", v.Features[1].ID, "feature 6 id") } func TestOffset(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?limit=2&offset=4") + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2&offset=4") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 2, len(v.Features), "# features") - equals(t, "5", v.Features[0].ID, "feature 5 id") - equals(t, "6", v.Features[1].ID, "feature 6 id") + util.Equals(t, 2, len(v.Features), "# features") + util.Equals(t, "5", v.Features[0].ID, "feature 5 id") + util.Equals(t, "6", v.Features[1].ID, "feature 6 id") } func TestOffsetInvalid(t *testing.T) { - doRequestStatus(t, "/collections/mock_a/items?offset=x", http.StatusBadRequest) + hTest.DoRequestStatus(t, "/collections/mock_a/items?offset=x", http.StatusBadRequest) } func TestTransformValid(t *testing.T) { - doRequest(t, "/collections/mock_a/items?transform=centroid") - doRequest(t, "/collections/mock_a/items?transform=ST_centroid") - doRequest(t, "/collections/mock_a/items?transform=st_centroid") - doRequest(t, "/collections/mock_a/items?transform=pointonsurface") - doRequest(t, "/collections/mock_a/items?transform=pointonsurface|centroid") + hTest.DoRequest(t, "/collections/mock_a/items?transform=centroid") + hTest.DoRequest(t, "/collections/mock_a/items?transform=ST_centroid") + hTest.DoRequest(t, "/collections/mock_a/items?transform=st_centroid") + hTest.DoRequest(t, "/collections/mock_a/items?transform=pointonsurface") + hTest.DoRequest(t, "/collections/mock_a/items?transform=pointonsurface|centroid") } func TestTransformInvalid(t *testing.T) { // envelope is not defined as a transform function - doRequestStatus(t, "/collections/mock_a/items?transform=envelope", http.StatusBadRequest) - doRequestStatus(t, "/collections/mock_a/items?transform=centroid|envelope", http.StatusBadRequest) + hTest.DoRequestStatus(t, "/collections/mock_a/items?transform=envelope", http.StatusBadRequest) + hTest.DoRequestStatus(t, "/collections/mock_a/items?transform=centroid|envelope", http.StatusBadRequest) } func TestBBox(t *testing.T) { - doRequest(t, "/collections/mock_a/items?bbox=1,2,3,4") + hTest.DoRequest(t, "/collections/mock_a/items?bbox=1,2,3,4") // TODO: add some tests } func TestBBoxInvalid(t *testing.T) { - doRequestStatus(t, "/collections/mock_a/items?bbox=1,2,3,x", http.StatusBadRequest) + hTest.DoRequestStatus(t, "/collections/mock_a/items?bbox=1,2,3,x", http.StatusBadRequest) } func TestProperties(t *testing.T) { // Tests: // - names are made unique (properties only include once) // - non-existing names are ignored - rr := doRequest(t, "/collections/mock_a/items?limit=2&properties=PROP_A,prop_c,prop_a,not_prop") + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2&properties=PROP_A,prop_c,prop_a,not_prop") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, 2, len(v.Features), "# features") - equals(t, 2, len(v.Features[0].Props), "feature 1 # properties") - equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") - equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") + util.Equals(t, 2, len(v.Features), "# features") + util.Equals(t, 2, len(v.Features[0].Props), "feature 1 # properties") + util.Equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") + util.Equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") } // TestPropertiesAll tests that no properties parameter returns all props func TestPropertiesAll(t *testing.T) { - rr := doRequest(t, "/collections/mock_a/items?limit=2") + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2") var v FeatureCollection - errUnMarsh := json.Unmarshal(readBody(rr), &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) // Note that JSON numbers are read as float64 - equals(t, 2, len(v.Features), "# features") - equals(t, 4, len(v.Features[0].Props), "feature 1 # properties") + util.Equals(t, 2, len(v.Features), "# features") + util.Equals(t, 4, len(v.Features[0].Props), "feature 1 # properties") - equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") - equals(t, 1.0, v.Features[0].Props["prop_b"], "feature 1 # property B") - equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") - equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D") + util.Equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") + util.Equals(t, 1.0, v.Features[0].Props["prop_b"], "feature 1 # property B") + util.Equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") + util.Equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D") } func TestCollectionNotFound(t *testing.T) { - doRequestStatus(t, "/collections/missing", http.StatusNotFound) + hTest.DoRequestStatus(t, "/collections/missing", http.StatusNotFound) } func TestCollectionMissingItemsNotFound(t *testing.T) { - doRequestStatus(t, "/collections/missing/items", http.StatusNotFound) + hTest.DoRequestStatus(t, "/collections/missing/items", http.StatusNotFound) } func TestFeatureNotFound(t *testing.T) { - doRequestStatus(t, "/collections/mock_a/items/999", http.StatusNotFound) + hTest.DoRequestStatus(t, "/collections/mock_a/items/999", http.StatusNotFound) } //============= Test functions func TestFunctionsJSON(t *testing.T) { path := "/functions" - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) var v api.FunctionsInfo errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") for i, fun := range catalogMock.FunctionDefs { checkFunctionSummary(t, v.Functions[i], fun) @@ -393,36 +368,36 @@ func TestFunctionJSON(t *testing.T) { } func TestFunctionNotFound(t *testing.T) { - doRequestStatus(t, "/functions/missing", http.StatusNotFound) + hTest.DoRequestStatus(t, "/functions/missing", http.StatusNotFound) } func TestFunctionMissingItemsNotFound(t *testing.T) { - doRequestStatus(t, "/functions/missing/items", http.StatusNotFound) + hTest.DoRequestStatus(t, "/functions/missing/items", http.StatusNotFound) } // ============ Test HTML generation // For now these just test that the template executes correctly // correctness/completess of HTML is not tested func TestHTMLRoot(t *testing.T) { - doRequest(t, "/index.html") + hTest.DoRequest(t, "/index.html") } func TestHTMLConformance(t *testing.T) { - doRequest(t, "/conformance.html") + hTest.DoRequest(t, "/conformance.html") } func TestHTMLCollections(t *testing.T) { - doRequest(t, "/collections.html") + hTest.DoRequest(t, "/collections.html") } func TestHTMLCollection(t *testing.T) { - doRequest(t, "/collections/mock_a.html") + hTest.DoRequest(t, "/collections/mock_a.html") } func TestHTMLItems(t *testing.T) { - doRequest(t, "/collections/mock_a/items.html") + hTest.DoRequest(t, "/collections/mock_a/items.html") } func TestHTMLItem(t *testing.T) { - doRequest(t, "/collections/mock_a/items/1.html") + hTest.DoRequest(t, "/collections/mock_a/items/1.html") } func TestHTMLFunctions(t *testing.T) { - rr := doRequest(t, "/functions.html") + rr := hTest.DoRequest(t, "/functions.html") for _, fun := range catalogMock.FunctionDefs { if !strings.Contains(rr.Body.String(), "http://test/functions/"+fun.Name+".json") { t.Errorf("Functions response should contain reference to " + fun.Name + ".json") @@ -430,86 +405,86 @@ func TestHTMLFunctions(t *testing.T) { } } func TestHTMLFunction(t *testing.T) { - doRequest(t, "/functions/fun_a.html") + hTest.DoRequest(t, "/functions/fun_a.html") } //=================================================== func checkCollection(tb testing.TB, coll *api.CollectionInfo, name string, title string) { - equals(tb, name, coll.Name, "Collection name") - equals(tb, title, coll.Title, "Collection title") + util.Equals(tb, name, coll.Name, "Collection name") + util.Equals(tb, title, coll.Title, "Collection title") path := "/collections/" + name - checkLink(tb, coll.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(tb, coll.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + checkLink(tb, coll.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(tb, coll.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") pathItems := path + "/items" - checkLink(tb, coll.Links[2], api.RelItems, api.ContentTypeGeoJSON, urlBase+pathItems) + checkLink(tb, coll.Links[2], api.RelItems, api.ContentTypeGeoJSON, hTest.UrlBase+pathItems) } func checkLink(tb testing.TB, link *api.Link, rel string, conType string, href string) { - equals(tb, rel, link.Rel, "Link rel") - equals(tb, conType, link.Type, "Link type") - equals(tb, href, link.Href, "Link href") + util.Equals(tb, rel, link.Rel, "Link rel") + util.Equals(tb, conType, link.Type, "Link type") + util.Equals(tb, href, link.Href, "Link href") } func checkFunctionSummary(tb testing.TB, v *api.FunctionSummary, fun *data.Function) { - equals(tb, fun.Name, v.Name, "Function name") - equals(tb, fun.Description, v.Description, "Function description") + util.Equals(tb, fun.Name, v.Name, "Function name") + util.Equals(tb, fun.Description, v.Description, "Function description") path := "/functions/" + fun.Name - checkLink(tb, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(tb, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + checkLink(tb, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(tb, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") pathItems := path + "/items" itemsType := api.ContentTypeJSON if fun.IsGeometryFunction() { itemsType = api.ContentTypeGeoJSON } - checkLink(tb, v.Links[2], api.RelItems, itemsType, urlBase+pathItems) + checkLink(tb, v.Links[2], api.RelItems, itemsType, hTest.UrlBase+pathItems) } func checkFunction(t *testing.T, fun *data.Function) { path := "/functions/" + fun.ID - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) var v api.FunctionInfo errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, fun.ID, v.Name, "Name") - equals(t, fun.Description, v.Description, "Description") + util.Equals(t, fun.ID, v.Name, "Name") + util.Equals(t, fun.Description, v.Description, "Description") //--- check parameters - assert(t, v.Parameters != nil, "Parameters property must be present") - equals(t, len(fun.InNames), len(v.Parameters), "Parameters len") + util.Assert(t, v.Parameters != nil, "Parameters property must be present") + util.Equals(t, len(fun.InNames), len(v.Parameters), "Parameters len") for i := 0; i < len(v.Parameters); i++ { - equals(t, fun.InNames[i], v.Parameters[i].Name, "Parameters[].Name") - equals(t, fun.InDbTypes[i], v.Parameters[i].Type, "Parameters[].Type") + util.Equals(t, fun.InNames[i], v.Parameters[i].Name, "Parameters[].Name") + util.Equals(t, fun.InDbTypes[i], v.Parameters[i].Type, "Parameters[].Type") } //--- check properties - assert(t, v.Properties != nil, "Properties property must be present") - equals(t, len(fun.OutNames), len(v.Properties), "Properties len") + util.Assert(t, v.Properties != nil, "Properties property must be present") + util.Equals(t, len(fun.OutNames), len(v.Properties), "Properties len") for i := 0; i < len(v.Properties); i++ { - equals(t, fun.OutNames[i], v.Properties[i].Name, "Properties[].Name") - equals(t, fun.OutJSONTypes[i], v.Properties[i].Type, "Properties[].Type") + util.Equals(t, fun.OutNames[i], v.Properties[i].Name, "Properties[].Name") + util.Equals(t, fun.OutJSONTypes[i], v.Properties[i].Type, "Properties[].Type") } //--- check links - checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, urlBase+path) - checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, urlBase+path+".html") + checkLink(t, v.Links[0], api.RelSelf, api.ContentTypeJSON, hTest.UrlBase+path) + checkLink(t, v.Links[1], api.RelAlt, api.ContentTypeHTML, hTest.UrlBase+path+".html") itemsType := api.ContentTypeJSON if fun.IsGeometryFunction() { itemsType = api.ContentTypeGeoJSON } - checkLink(t, v.Links[2], api.RelItems, itemsType, urlBase+path+"/items") + checkLink(t, v.Links[2], api.RelItems, itemsType, hTest.UrlBase+path+"/items") } // check if item is available and is not empty func checkItem(t *testing.T, id int) { path := fmt.Sprintf("/collections/mock_a/items/%d", id) - resp := doRequest(t, path) + resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) // extracted from catalog_db.go @@ -522,10 +497,10 @@ func checkItem(t *testing.T, id int) { var v featureData errUnMarsh := json.Unmarshal(body, &v) - assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) - equals(t, "Feature", v.Type, "feature type") + util.Equals(t, "Feature", v.Type, "feature type") actId, _ := strconv.Atoi(v.ID) - equals(t, id, actId, "feature id") - equals(t, 4, len(v.Props), "# feature props") + util.Equals(t, id, actId, "feature id") + util.Equals(t, 4, len(v.Props), "# feature props") } diff --git a/internal/service/service.go b/internal/service/service.go index 227711aa..c2d99db8 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -13,7 +13,6 @@ import ( "github.com/CrunchyData/pg_featureserv/internal/conf" "github.com/CrunchyData/pg_featureserv/internal/data" "github.com/gorilla/handlers" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -31,7 +30,6 @@ import ( */ var catalogInstance data.Catalog -var router *mux.Router var server *http.Server var isTLSEnabled bool var serverTLS *http.Server @@ -55,7 +53,7 @@ func createServers() { } log.Infof("CORS Allowed Origins: %v\n", conf.Configuration.Server.CORSOrigins) - router = initRouter(confServ.BasePath) + router := InitRouter(confServ.BasePath) // writeTimeout is slighlty longer than request timeout to allow writing error response timeoutSecRequest := conf.Configuration.Server.WriteTimeoutSec @@ -99,6 +97,11 @@ func createServers() { } } +// Set catalog instance +func SetCatalogInstance(catalog data.Catalog) { + catalogInstance = catalog +} + // Serve starts the web service func Serve(catalog data.Catalog) { confServ := conf.Configuration.Server diff --git a/util/test_common.go b/util/test_common.go new file mode 100644 index 00000000..8cfc22bb --- /dev/null +++ b/util/test_common.go @@ -0,0 +1,51 @@ +package util + +/* + Copyright 2022 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ( + "fmt" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +//---- testing utilities from https://github.com/benbjohnson/testing + +// assert fails the test if the condition is false. +func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { + if !condition { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) + tb.FailNow() + } +} + +// equals fails the test if exp is not equal to act. +func Equals(tb testing.TB, exp, act interface{}, msg string) { + if !reflect.DeepEqual(exp, act) { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("%s:%d: %s - expected: %#v; got: %#v\n", filepath.Base(file), line, msg, exp, act) + tb.FailNow() + } +} + +// isError fails the test if err is nil, ie. test should have failed! +func AssertIsError(tb testing.TB, err error, msg string) { + if err == nil { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("%s:%d: %s - expected error\n", filepath.Base(file), line, msg) + tb.FailNow() + } +} diff --git a/internal/service/handler_common_test.go b/util/test_http.go similarity index 51% rename from internal/service/handler_common_test.go rename to util/test_http.go index 4a128f59..8365667b 100644 --- a/internal/service/handler_common_test.go +++ b/util/test_http.go @@ -1,4 +1,4 @@ -package service +package util /* Copyright 2022 Crunchy Data Solutions, Inc. @@ -19,38 +19,78 @@ import ( "io/ioutil" "net/http" "net/http/httptest" - "path/filepath" - "reflect" - "runtime" "testing" + + "github.com/CrunchyData/pg_featureserv/internal/conf" + "github.com/gorilla/mux" ) +type HttpTesting struct { + UrlBase string + BasePath string + Router *mux.Router +} + +func MakeHttpTesting(urlBase string, basePath string, router *mux.Router) HttpTesting { + hTest := HttpTesting{ + UrlBase: urlBase, + BasePath: basePath, + Router: router, + } + hTest.Setup() + return hTest +} + +func (hTest *HttpTesting) Setup() { + conf.Configuration = conf.Config{ + Server: conf.Server{ + HttpHost: "0.0.0.0", + HttpPort: 9000, + UrlBase: hTest.UrlBase, + BasePath: hTest.BasePath, + AssetsPath: "../../assets", + TransformFunctions: []string{ + "ST_Centroid", + "ST_PointOnSurface", + }, + }, + Paging: conf.Paging{ + LimitDefault: 10, + LimitMax: 1000, + }, + Metadata: conf.Metadata{ + Title: "test", + Description: "test", + }, + } +} + // returns the all the body from an http response -func readBody(resp *httptest.ResponseRecorder) []byte { +func (hTest *HttpTesting) ReadBody(resp *httptest.ResponseRecorder) []byte { body, _ := ioutil.ReadAll(resp.Body) return body } // do an http request to url with default method GET and expected status OK -func doRequest(t *testing.T, url string) *httptest.ResponseRecorder { - return doRequestStatus(t, url, http.StatusOK) +func (hTest *HttpTesting) DoRequest(t *testing.T, url string) *httptest.ResponseRecorder { + return hTest.DoRequestStatus(t, url, http.StatusOK) } -func doPostRequest(t *testing.T, url string, data []byte, header http.Header) *httptest.ResponseRecorder { - return doRequestMethodStatus(t, "POST", url, data, header, http.StatusCreated) +func (hTest *HttpTesting) DoPostRequest(t *testing.T, url string, data []byte, header http.Header) *httptest.ResponseRecorder { + return hTest.DoRequestMethodStatus(t, "POST", url, data, header, http.StatusCreated) } // do an http request to url with default method GET and a specific expected status -func doRequestStatus(t *testing.T, url string, +func (hTest *HttpTesting) DoRequestStatus(t *testing.T, url string, statusExpected int) *httptest.ResponseRecorder { - return doRequestMethodStatus(t, "GET", url, nil, nil, statusExpected) + return hTest.DoRequestMethodStatus(t, "GET", url, nil, nil, statusExpected) } // do an http request to url with a specific method and specific expected status -func doRequestMethodStatus(t *testing.T, method string, url string, +func (hTest *HttpTesting) DoRequestMethodStatus(t *testing.T, method string, url string, data []byte, header http.Header, statusExpected int) *httptest.ResponseRecorder { - req, err := http.NewRequest(method, basePath+url, bytes.NewReader(data)) + req, err := http.NewRequest(method, hTest.BasePath+url, bytes.NewReader(data)) if err != nil { t.Fatal(err) } @@ -60,7 +100,7 @@ func doRequestMethodStatus(t *testing.T, method string, url string, } rr := httptest.NewRecorder() - router.ServeHTTP(rr, req) + hTest.Router.ServeHTTP(rr, req) // Check the status code status := rr.Code @@ -72,34 +112,14 @@ func doRequestMethodStatus(t *testing.T, method string, url string, } if bodyMsg != "" { - equals(t, + Equals(t, statusExpected, status, fmt.Errorf("handler returned wrong status code.\n\tCaused by: %v", bodyMsg).Error()) } else { - equals(t, + Equals(t, statusExpected, status, "handler returned wrong status code.") } } return rr } - -//---- testing utilities from https://github.com/benbjohnson/testing - -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() - } -} - -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}, msg string) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("%s:%d: %s - expected: %#v; got: %#v\n", filepath.Base(file), line, msg, exp, act) - tb.FailNow() - } -} From 9e2994494d2aeed4644770283d6d8037ca5d4bb0 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Fri, 2 Sep 2022 15:32:23 +0200 Subject: [PATCH 10/13] feat(CREATE): implements AddTableFeature in catalog_db --- internal/api/api.go | 5 +- internal/data/catalog_db.go | 75 +++++++++- internal/data/catalog_db_fun.go | 2 +- internal/data/catalog_mock.go | 8 +- internal/service/db_test/handler_db_test.go | 149 ++++++++++++++++++++ internal/service/handler.go | 22 +-- util/test_db.go | 100 +++++++++++++ 7 files changed, 338 insertions(+), 23 deletions(-) create mode 100644 internal/service/db_test/handler_db_test.go create mode 100644 util/test_db.go diff --git a/internal/api/api.go b/internal/api/api.go index f88f1c88..211e48a4 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -82,9 +82,9 @@ const ( ErrMsgFunctionNotFound = "Function not found: %v" ErrMsgFunctionAccess = "Unable to access Function: %v" ErrMsgInvalidParameterValue = "Invalid value for parameter %v: %v" - ErrMsgInvalidQuery = "Invalid query parameters" + ErrMsgInvalidQuery = "Invalid query parameters" ErrMsgDataReadError = "Unable to read data from: %v" - ErrMsgDataWriteError = "Unable to write data to: %v" + ErrMsgDataWriteError = "Unable to write data to: %v" ErrMsgNoDataRead = "No data read from: %v" ErrMsgRequestTimeout = "Maximum time exceeded. Request cancelled." ) @@ -645,6 +645,7 @@ func PathItem(name string, fid string) string { var Db2OpenapiFormatMap = map[string]string{ "int": "integer", + "int4": "integer", "long": "int64", "text": "string", } diff --git a/internal/data/catalog_db.go b/internal/data/catalog_db.go index fadacbb8..ea5490ba 100644 --- a/internal/data/catalog_db.go +++ b/internal/data/catalog_db.go @@ -156,8 +156,8 @@ func (cat *catalogDB) Tables() ([]*Table, error) { } func (cat *catalogDB) TableReload(name string) { - tbl, ok := cat.tableMap[name] - if !ok { + tbl, err := cat.TableByName(name) + if err != nil { return } // load extent (which may change over time @@ -197,7 +197,11 @@ func (cat *catalogDB) TableByName(name string) (*Table, error) { cat.refreshTables(false) tbl, ok := cat.tableMap[name] if !ok { - return nil, nil + tbl, ok := cat.tableMap["public."+name] + if !ok { + return nil, nil + } + return tbl, nil } return tbl, nil } @@ -238,7 +242,60 @@ func (cat *catalogDB) TableFeature(ctx context.Context, name string, id string, } func (cat *catalogDB) AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) { - panic("catalogDB::AddTableFeature unimplemented") + var schemaObject geojsonFeatureData + err := json.Unmarshal(jsonData, &schemaObject) + if err != nil { + return -9999, err + } + var columnStr string + var placementStr string + var values []interface{} + + tbl, err := cat.TableByName(tableName) + if err != nil { + return -9999, err + } + var i = 0 + for c, t := range tbl.DbTypes { + if c == tbl.IDColumn { + continue // ignore id column + } + + i++ + columnStr += c + placementStr += fmt.Sprintf("$%d", i) + if t.Type == "int4" { + values = append(values, int(schemaObject.Props[c].(float64))) + } else { + values = append(values, schemaObject.Props[c]) + } + + if i < len(tbl.Columns)-1 { + columnStr += ", " + placementStr += ", " + } + + } + + i++ + columnStr += ", " + tbl.GeometryColumn + placementStr += fmt.Sprintf(", ST_GeomFromGeoJSON($%d)", i) + geomJson, _ := schemaObject.Geom.MarshalJSON() + values = append(values, geomJson) + + sqlStatement := fmt.Sprintf(` + INSERT INTO %s (%s) + VALUES (%s) + RETURNING %s`, + tbl.ID, columnStr, placementStr, tbl.IDColumn) + + var id int64 = -1 + err = cat.dbconn.QueryRow(ctx, sqlStatement, values...).Scan(&id) + if err != nil { + return -9999, err + } + + return id, nil } func (cat *catalogDB) refreshTables(force bool) { @@ -392,7 +449,6 @@ func readFeatures(ctx context.Context, db *pgxpool.Pool, sql string, idColIndex return readFeaturesWithArgs(ctx, db, sql, nil, idColIndex, propCols) } -//nolint:unused func readFeaturesWithArgs(ctx context.Context, db *pgxpool.Pool, sql string, args []interface{}, idColIndex int, propCols []string) ([]string, error) { start := time.Now() rows, err := db.Query(ctx, sql, args...) @@ -448,7 +504,7 @@ func scanFeature(rows pgx.Rows, idColIndex int, propNames []string) string { id = fmt.Sprintf("%v", vals[idColIndex+propOffset]) } - props := extractProperties(vals, propOffset, propNames) + props := extractProperties(vals, idColIndex, propOffset, propNames) //--- geom value is expected to be a GeoJSON string or geojson object //--- convert NULL to an empty string @@ -463,9 +519,12 @@ func scanFeature(rows pgx.Rows, idColIndex int, propNames []string) string { } } -func extractProperties(vals []interface{}, propOffset int, propNames []string) map[string]interface{} { +func extractProperties(vals []interface{}, idColIndex int, propOffset int, propNames []string) map[string]interface{} { props := make(map[string]interface{}) for i, name := range propNames { + if i == idColIndex { + continue + } // offset vals index by 2 to skip geom, id props[name] = toJSONValue(vals[i+propOffset]) //fmt.Printf("%v: %v\n", name, val) @@ -594,6 +653,7 @@ func makeFeatureJSON(id string, geom string, props map[string]interface{}) strin return jsonStr } +// TODO should be exported in catalog.go type geojsonFeatureData struct { Type string `json:"type"` ID string `json:"id,omitempty"` @@ -601,6 +661,7 @@ type geojsonFeatureData struct { Props map[string]interface{} `json:"properties"` } +// TODO should be exported in catalog.go func makeGeojsonFeatureJSON(id string, geom geojson.Geometry, props map[string]interface{}) string { featData := geojsonFeatureData{ Type: "Feature", diff --git a/internal/data/catalog_db_fun.go b/internal/data/catalog_db_fun.go index 3c03fa25..f7eb1009 100644 --- a/internal/data/catalog_db_fun.go +++ b/internal/data/catalog_db_fun.go @@ -282,6 +282,6 @@ func scanDataRow(rows pgx.Rows, hasID bool, propNames []string) map[string]inter //fmt.Println(vals) //fmt.Println(geom) - props := extractProperties(vals, 0, propNames) + props := extractProperties(vals, -1, 0, propNames) return props } diff --git a/internal/data/catalog_mock.go b/internal/data/catalog_mock.go index 04872ffc..e1a03a37 100644 --- a/internal/data/catalog_mock.go +++ b/internal/data/catalog_mock.go @@ -92,9 +92,9 @@ func newCatalogMock() CatalogMock { } tableData := map[string][]*featureMock{} - tableData["mock_a"] = makePointFeatures(layerA.Extent, 3, 3) - tableData["mock_b"] = makePointFeatures(layerB.Extent, 10, 10) - tableData["mock_c"] = makePointFeatures(layerC.Extent, 100, 100) + tableData["mock_a"] = MakePointFeatures(layerA.Extent, 3, 3) + tableData["mock_b"] = MakePointFeatures(layerB.Extent, 10, 10) + tableData["mock_c"] = MakePointFeatures(layerC.Extent, 100, 100) var tables []*Table tables = append(tables, layerA) @@ -306,7 +306,7 @@ func MakeFeatureMockPointAsJSON(id int, x float64, y float64, columns []string) return feat.toJSON(columns) } -func makePointFeatures(extent Extent, nx int, ny int) []*featureMock { +func MakePointFeatures(extent Extent, nx int, ny int) []*featureMock { basex := extent.Minx basey := extent.Miny dx := (extent.Maxx - extent.Minx) / float64(nx) diff --git a/internal/service/db_test/handler_db_test.go b/internal/service/db_test/handler_db_test.go new file mode 100644 index 00000000..9f2afc4b --- /dev/null +++ b/internal/service/db_test/handler_db_test.go @@ -0,0 +1,149 @@ +package db_test + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "testing" + + "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/CrunchyData/pg_featureserv/internal/data" + "github.com/CrunchyData/pg_featureserv/internal/service" + "github.com/CrunchyData/pg_featureserv/util" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/paulmach/orb/geojson" +) + +var hTest util.HttpTesting +var db *pgxpool.Pool +var cat data.Catalog + +// extracted from catalog_db.go +// TODO should be imported from catalog.go +type geojsonFeatureData struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Geom *geojson.Geometry `json:"geometry"` + Props map[string]interface{} `json:"properties"` +} + +func TestMain(m *testing.M) { + db = util.CreateTestDb() + defer util.CloseTestDb(db) + + cat = data.CatDBInstance() + service.SetCatalogInstance(cat) + + hTest = util.MakeHttpTesting("http://test", "/pg_featureserv", service.InitRouter("/pg_featureserv")) + service.Initialize() + + os.Exit(m.Run()) +} + +func TestProperDbInit(t *testing.T) { + tables, _ := cat.Tables() + util.Equals(t, 2, len(tables), "# table in DB") +} + +func TestTestPropertiesAllFromDb(t *testing.T) { + /*rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2") + + var v FeatureCollection + errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + + // Note that JSON numbers are read as float64 + util.Equals(t, 2, len(v.Features), "# features") + util.Equals(t, 4, len(v.Features[0].Props), "feature 1 # properties") + + util.Equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") + util.Equals(t, 1.0, v.Features[0].Props["prop_b"], "feature 1 # property B") + util.Equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") + util.Equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D")*/ +} + +func TestCreateFeatureWithBadGeojsonInputDb(t *testing.T) { + var header = make(http.Header) + header.Add("Content-Type", "application/geo+json") + + jsonStr := `[{ + "id": 101, + "name": "Test", + "email": "test@test.com" + }, { + "id": 102, + "name": "Sample", + "email": "sample@test.com" + }]` + + rr := hTest.DoRequestMethodStatus(t, "POST", "/collections/mock_a/items", []byte(jsonStr), header, http.StatusInternalServerError) + + util.Equals(t, http.StatusInternalServerError, rr.Code, "Should have failed") + util.Assert(t, strings.Index(rr.Body.String(), fmt.Sprintf(api.ErrMsgCreateFeatureNotConform+"\n", "mock_a")) == 0, "Should have failed with not conform") +} + +func TestCreateFeatureDb(t *testing.T) { + var header = make(http.Header) + header.Add("Content-Type", "application/geo+json") + + //--- retrieve max feature id before insert + var features []string + params := data.QueryParam{Limit: 100, Offset: 0, Crs: 4326} + features, _ = cat.TableFeatures(context.Background(), "mock_a", ¶ms) + maxIdBefore := len(features) + + //--- generate json from new object + tables, _ := cat.Tables() + var cols []string + for _, t := range tables { + if t.ID == "public.mock_a" { + for _, c := range t.Columns { + if c != "id" { + cols = append(cols, c) + } + } + break + } + } + jsonStr := data.MakeFeatureMockPointAsJSON(99, 12, 34, cols) + fmt.Println(jsonStr) + + // -- do the request call but we have to force the catalogInstance to db during this operation + rr := hTest.DoPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header) + + loc := rr.Header().Get("Location") + + //--- retrieve max feature id after insert + features, _ = cat.TableFeatures(context.Background(), "mock_a", ¶ms) + maxIdAfter := len(features) + + util.Assert(t, maxIdAfter > maxIdBefore, "# feature in db") + util.Assert(t, len(loc) > 1, "Header location must not be empty") + util.Equals(t, fmt.Sprintf("http://test/collections/mock_a/items/%d", maxIdAfter), loc, + "Header location must contain valid data") + + // check if it can be read + checkItem(t, maxIdAfter) +} + +// check if item is available and is not empty +// copy from service/handler_test.go +func checkItem(t *testing.T, id int) { + path := fmt.Sprintf("/collections/mock_a/items/%d", id) + resp := hTest.DoRequest(t, path) + body, _ := ioutil.ReadAll(resp.Body) + + var v geojsonFeatureData + errUnMarsh := json.Unmarshal(body, &v) + util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) + + util.Equals(t, "Feature", v.Type, "feature type") + actId, _ := strconv.Atoi(v.ID) + util.Equals(t, id, actId, "feature id") + util.Equals(t, 4, len(v.Props), "# feature props") +} diff --git a/internal/service/handler.go b/internal/service/handler.go index 478bdd2a..4cd032fb 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -435,7 +435,9 @@ func getCreateItemSchema(ctx context.Context, table *data.Table) (openapi3.Schem requiredTypeKeys := make([]string, 0, len(table.DbTypes)) for k := range table.DbTypes { - requiredTypeKeys = append(requiredTypeKeys, k) + if k != table.IDColumn { + requiredTypeKeys = append(requiredTypeKeys, k) + } } sort.Strings(requiredTypeKeys) @@ -451,14 +453,16 @@ func getCreateItemSchema(ctx context.Context, table *data.Table) (openapi3.Schem // update properties by their name and type props.Properties = make(map[string]*openapi3.SchemaRef) for k, v := range table.DbTypes { - propType := v.Type - if api.Db2OpenapiFormatMap[v.Type] != "" { - propType = api.Db2OpenapiFormatMap[v.Type] - } - props.Properties[k] = &openapi3.SchemaRef{ - Value: &openapi3.Schema{ - Type: propType, - }, + if k != table.IDColumn { + propType := v.Type + if api.Db2OpenapiFormatMap[v.Type] != "" { + propType = api.Db2OpenapiFormatMap[v.Type] + } + props.Properties[k] = &openapi3.SchemaRef{ + Value: &openapi3.Schema{ + Type: propType, + }, + } } } diff --git a/util/test_db.go b/util/test_db.go new file mode 100644 index 00000000..c2ceeca3 --- /dev/null +++ b/util/test_db.go @@ -0,0 +1,100 @@ +package util + +/* + Copyright 2022 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ( + "context" + "os" + + "github.com/CrunchyData/pg_featureserv/internal/conf" + "github.com/CrunchyData/pg_featureserv/internal/data" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + log "github.com/sirupsen/logrus" +) + +func CreateTestDb() *pgxpool.Pool { + dbURL := os.Getenv(conf.AppConfig.EnvDBURL) + if dbURL == "" { + dbURL = "postgresql://postgres@localhost/pg_featureserv" + log.Warnf("No env var '%s' defined, using default value: %s", conf.AppConfig.EnvDBURL, dbURL) + } + conf.Configuration.Database.DbConnection = dbURL + conf.Configuration.Database.DbPoolMaxConnLifeTime = "1h" + + ctx := context.Background() + dbconfig, errConf := pgxpool.ParseConfig(conf.Configuration.Database.DbConnection) + if errConf != nil { + log.Fatal(errConf) + } + db, errConn := pgxpool.ConnectConfig(ctx, dbconfig) + if errConn != nil { + log.Fatal(errConn) + } + + dbName := dbconfig.ConnConfig.Config.Database + dbUser := dbconfig.ConnConfig.Config.User + dbHost := dbconfig.ConnConfig.Config.Host + log.Debugf("Connected as %s to %s @ %s", dbUser, dbName, dbHost) + + // same as featureMock + _, errExec := db.Exec(ctx, ` + DROP TABLE IF EXISTS mock_a CASCADE; + CREATE TABLE IF NOT EXISTS public.mock_a ( + id SERIAL PRIMARY KEY, + geometry public.geometry(Point, 4326), + prop_a text, + prop_b int, + prop_c text, + prop_d int + ); + `) + if errExec != nil { + CloseTestDb(db) + log.Fatal(errExec) + } + + feats := data.MakePointFeatures(data.Extent{Minx: -120, Miny: 40, Maxx: -74, Maxy: 50}, + 3, 3) + b := &pgx.Batch{} + sqlStatement := ` + INSERT INTO public.mock_a (geometry, prop_a, prop_b, prop_c, prop_d) + VALUES (ST_GeomFromGeoJSON($1), $2, $3, $4, $5)` + + for _, f := range feats { + geomStr, _ := f.Geom.MarshalJSON() + b.Queue(sqlStatement, geomStr, f.PropA, f.PropB, f.PropC, f.PropD) + } + res := db.SendBatch(ctx, b) + if res == nil { + CloseTestDb(db) + log.Fatal("Injection failed") + } + res.Close() + + log.Debugf("Sample data injected") + + return db +} + +func CloseTestDb(db *pgxpool.Pool) { + log.Debugf("Sample db will be cleared...") + _, errExec := db.Exec(context.Background(), "DROP TABLE IF EXISTS public.mock_a CASCADE;") + if errExec != nil { + log.Warnf("Failed to drop sample db! ") + log.Warnf(errExec.Error()) + } + db.Close() + log.Debugf("Sample db cleared!") +} From 7831538fb3d640f1f266caa7ccdcab520a1045f4 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Mon, 5 Sep 2022 09:53:27 +0200 Subject: [PATCH 11/13] move common type structs to test_common.go --- internal/service/db_test/handler_db_test.go | 20 +++------- internal/service/handler_test.go | 44 ++++++--------------- util/test_common.go | 23 +++++++++++ 3 files changed, 41 insertions(+), 46 deletions(-) diff --git a/internal/service/db_test/handler_db_test.go b/internal/service/db_test/handler_db_test.go index 9f2afc4b..be6fef29 100644 --- a/internal/service/db_test/handler_db_test.go +++ b/internal/service/db_test/handler_db_test.go @@ -16,22 +16,12 @@ import ( "github.com/CrunchyData/pg_featureserv/internal/service" "github.com/CrunchyData/pg_featureserv/util" "github.com/jackc/pgx/v4/pgxpool" - "github.com/paulmach/orb/geojson" ) var hTest util.HttpTesting var db *pgxpool.Pool var cat data.Catalog -// extracted from catalog_db.go -// TODO should be imported from catalog.go -type geojsonFeatureData struct { - Type string `json:"type"` - ID string `json:"id,omitempty"` - Geom *geojson.Geometry `json:"geometry"` - Props map[string]interface{} `json:"properties"` -} - func TestMain(m *testing.M) { db = util.CreateTestDb() defer util.CloseTestDb(db) @@ -50,10 +40,10 @@ func TestProperDbInit(t *testing.T) { util.Equals(t, 2, len(tables), "# table in DB") } -func TestTestPropertiesAllFromDb(t *testing.T) { - /*rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2") +func TestPropertiesAllFromDb(t *testing.T) { + rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -64,7 +54,7 @@ func TestTestPropertiesAllFromDb(t *testing.T) { util.Equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A") util.Equals(t, 1.0, v.Features[0].Props["prop_b"], "feature 1 # property B") util.Equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C") - util.Equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D")*/ + util.Equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D") } func TestCreateFeatureWithBadGeojsonInputDb(t *testing.T) { @@ -138,7 +128,7 @@ func checkItem(t *testing.T, id int) { resp := hTest.DoRequest(t, path) body, _ := ioutil.ReadAll(resp.Body) - var v geojsonFeatureData + var v util.GeojsonFeatureData errUnMarsh := json.Unmarshal(body, &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index 86cfddfb..c784f796 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -28,24 +28,6 @@ import ( "github.com/CrunchyData/pg_featureserv/util" ) -// Define a FeatureCollection structure for parsing test data - -type Feature struct { - Type string `json:"type"` - ID string `json:"id,omitempty"` - Geom *json.RawMessage `json:"geometry"` - Props map[string]interface{} `json:"properties"` -} - -type FeatureCollection struct { - Type string `json:"type"` - Features []*Feature `json:"features"` - NumberMatched uint `json:"numberMatched,omitempty"` - NumberReturned uint `json:"numberReturned"` - TimeStamp string `json:"timeStamp,omitempty"` - Links []*api.Link `json:"links"` -} - var hTest util.HttpTesting var catalogMock *data.CatalogMock @@ -149,7 +131,7 @@ func TestCollectionItem(t *testing.T) { func TestFilterB(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?prop_b=1") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -159,7 +141,7 @@ func TestFilterB(t *testing.T) { func TestFilterD(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_d=1") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -169,7 +151,7 @@ func TestFilterD(t *testing.T) { func TestFilterBD(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_b=2&prop_d=2") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -179,7 +161,7 @@ func TestFilterBD(t *testing.T) { func TestFilterBDNone(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_c/items?prop_b=1&prop_d=2") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -189,7 +171,7 @@ func TestFilterBDNone(t *testing.T) { func TestSortBy(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=prop_b") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -199,7 +181,7 @@ func TestSortBy(t *testing.T) { func TestSortByDesc(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=-prop_b") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -209,7 +191,7 @@ func TestSortByDesc(t *testing.T) { func TestSortByAsc(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?sortby=+prop_b") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -219,7 +201,7 @@ func TestSortByAsc(t *testing.T) { func TestLimit(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=3") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -232,7 +214,7 @@ func TestLimit(t *testing.T) { func TestLimitZero(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=0") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -247,7 +229,7 @@ func TestLimitInvalid(t *testing.T) { func TestQueryParamCase(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?LIMIT=2&Offset=4") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -259,7 +241,7 @@ func TestQueryParamCase(t *testing.T) { func TestOffset(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2&offset=4") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -301,7 +283,7 @@ func TestProperties(t *testing.T) { // - non-existing names are ignored rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2&properties=PROP_A,prop_c,prop_a,not_prop") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) @@ -315,7 +297,7 @@ func TestProperties(t *testing.T) { func TestPropertiesAll(t *testing.T) { rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2") - var v FeatureCollection + var v util.FeatureCollection errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v) util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh)) diff --git a/util/test_common.go b/util/test_common.go index 8cfc22bb..e2799287 100644 --- a/util/test_common.go +++ b/util/test_common.go @@ -19,8 +19,31 @@ import ( "reflect" "runtime" "testing" + + "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/paulmach/orb/geojson" ) +// extracted from catalog_db.go +// TODO should be imported from catalog.go +type GeojsonFeatureData struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Geom *geojson.Geometry `json:"geometry"` + Props map[string]interface{} `json:"properties"` +} + +// Define a FeatureCollection structure for parsing test data +// TODO should be move to and imported from catalog.go +type FeatureCollection struct { + Type string `json:"type"` + Features []*GeojsonFeatureData `json:"features"` + NumberMatched uint `json:"numberMatched,omitempty"` + NumberReturned uint `json:"numberReturned"` + TimeStamp string `json:"timeStamp,omitempty"` + Links []*api.Link `json:"links"` +} + //---- testing utilities from https://github.com/benbjohnson/testing // assert fails the test if the condition is false. From fdfb738355d1f8b61016009f460c6a336c384a95 Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Mon, 10 Oct 2022 18:02:40 +0200 Subject: [PATCH 12/13] feat(CREATE): add configuration option to enable/disable database writes --- config/pg_featureserv.toml.example | 3 +++ hugo/content/installation/configuration.md | 3 +++ internal/conf/config.go | 2 ++ internal/service/db_test/handler_db_test.go | 3 +++ internal/service/handler.go | 7 +++++-- internal/service/handler_test.go | 3 +++ 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/config/pg_featureserv.toml.example b/config/pg_featureserv.toml.example index cb35560f..b8ade966 100644 --- a/config/pg_featureserv.toml.example +++ b/config/pg_featureserv.toml.example @@ -68,6 +68,9 @@ WriteTimeoutSec = 30 # Publish functions from these schemas (default is publish postgisftw) # FunctionIncludes = [ "postgisftw", "schema2" ] +# Allow write changes to database. Default is to read only. +# AllowWrite = false + [Paging] # The default number of features in a response LimitDefault = 20 diff --git a/hugo/content/installation/configuration.md b/hugo/content/installation/configuration.md index f0b3273e..116991a0 100644 --- a/hugo/content/installation/configuration.md +++ b/hugo/content/installation/configuration.md @@ -119,6 +119,9 @@ WriteTimeoutSec = 30 # Publish functions from these schemas (default is publish postgisftw) # FunctionIncludes = [ "postgisftw", "schema2" ] +# Allow write changes to database. Default is to read only. +# AllowWrite = false + [Paging] # The default number of features in a response LimitDefault = 20 diff --git a/internal/conf/config.go b/internal/conf/config.go index e304cd38..48ce8f5d 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -44,6 +44,7 @@ func setDefaultConfig() { viper.SetDefault("Database.TableIncludes", []string{}) viper.SetDefault("Database.TableExcludes", []string{}) viper.SetDefault("Database.FunctionIncludes", []string{"postgisftw"}) + viper.SetDefault("Database.AllowWrite", false) viper.SetDefault("Paging.LimitDefault", 10) viper.SetDefault("Paging.LimitMax", 1000) @@ -94,6 +95,7 @@ type Database struct { TableIncludes []string TableExcludes []string FunctionIncludes []string + AllowWrite bool } // Metadata config diff --git a/internal/service/db_test/handler_db_test.go b/internal/service/db_test/handler_db_test.go index be6fef29..aaad13f2 100644 --- a/internal/service/db_test/handler_db_test.go +++ b/internal/service/db_test/handler_db_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/CrunchyData/pg_featureserv/internal/conf" "github.com/CrunchyData/pg_featureserv/internal/data" "github.com/CrunchyData/pg_featureserv/internal/service" "github.com/CrunchyData/pg_featureserv/util" @@ -23,6 +24,8 @@ var db *pgxpool.Pool var cat data.Catalog func TestMain(m *testing.M) { + conf.Configuration.Database.AllowWrite = true + db = util.CreateTestDb() defer util.CloseTestDb(db) diff --git a/internal/service/handler.go b/internal/service/handler.go index 4cd032fb..78c3116e 100644 --- a/internal/service/handler.go +++ b/internal/service/handler.go @@ -61,9 +61,12 @@ func InitRouter(basePath string) *mux.Router { addRoute(router, "/collections/{id}/items", handleCollectionItems) addRoute(router, "/collections/{id}/items.{fmt}", handleCollectionItems) - addRouteWithMethod(router, "/collections/{id}/items", handleCreateCollectionItem, "POST") - addRoute(router, "/collections/{id}/schema", handleCollectionSchemas) + if conf.Configuration.Database.AllowWrite { + addRouteWithMethod(router, "/collections/{id}/items", handleCreateCollectionItem, "POST") + + addRoute(router, "/collections/{id}/schema", handleCollectionSchemas) + } addRoute(router, "/collections/{id}/items/{fid}", handleItem) addRoute(router, "/collections/{id}/items/{fid}.{fmt}", handleItem) diff --git a/internal/service/handler_test.go b/internal/service/handler_test.go index c784f796..1da32587 100644 --- a/internal/service/handler_test.go +++ b/internal/service/handler_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/CrunchyData/pg_featureserv/internal/api" + "github.com/CrunchyData/pg_featureserv/internal/conf" "github.com/CrunchyData/pg_featureserv/internal/data" "github.com/CrunchyData/pg_featureserv/util" ) @@ -33,6 +34,8 @@ var hTest util.HttpTesting var catalogMock *data.CatalogMock func TestMain(m *testing.M) { + conf.Configuration.Database.AllowWrite = true + catalogMock = data.CatMockInstance() catalogInstance = catalogMock From 36836ef27ea15429093dbb9005806838cdfc17bf Mon Sep 17 00:00:00 2001 From: bdm-oslandia Date: Thu, 8 Sep 2022 10:12:21 +0200 Subject: [PATCH 13/13] add missing /collections/{collectionId}/items POST api * openapi.go: add paramFeatureID --- internal/api/openapi.go | 114 +++++++++++++++++++++++--- internal/service/handler_post_test.go | 14 +++- 2 files changed, 117 insertions(+), 11 deletions(-) diff --git a/internal/api/openapi.go b/internal/api/openapi.go index 5a0bf7c3..ba266fbb 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -14,6 +14,7 @@ package api */ import ( + "encoding/json" "net/url" "github.com/CrunchyData/pg_featureserv/internal/conf" @@ -21,6 +22,68 @@ import ( log "github.com/sirupsen/logrus" ) +func getFeatureExample() map[string]interface{} { + var result map[string]interface{} + var jsonStr = `{"type":"Feature","geometry":{"type":"Point","coordinates":[-70.88461956597838,47.807897059236495]},"properties":{"prop_a":"propA","prop_b":1,"prop_c":"propC","prop_d":1}}` + err := json.Unmarshal([]byte(jsonStr), &result) + if err != nil { + return nil + } + return result +} + +var FeatureSchema openapi3.Schema = openapi3.Schema{ + Type: "object", + Required: []string{}, + Properties: map[string]*openapi3.SchemaRef{ + "id": { + Value: &openapi3.Schema{ + OneOf: []*openapi3.SchemaRef{ + openapi3.NewSchemaRef("", &openapi3.Schema{ + Type: "number", Format: "long", + }), + openapi3.NewSchemaRef("", &openapi3.Schema{ + Type: "string", + }), + }, + }, + }, + "type": { + Value: &openapi3.Schema{ + Type: "string", + Default: "Feature", + }, + }, + "geometry": { + Value: &openapi3.Schema{ + Type: "string", // mandatory to validate the schema + OneOf: []*openapi3.SchemaRef{ + openapi3.NewSchemaRef("https://geojson.org/schema/Point.json", &openapi3.Schema{Type: "string"}), + openapi3.NewSchemaRef("https://geojson.org/schema/LineString.json", &openapi3.Schema{Type: "string"}), + openapi3.NewSchemaRef("https://geojson.org/schema/Polygon.json", &openapi3.Schema{Type: "string"}), + openapi3.NewSchemaRef("https://geojson.org/schema/MultiPoint.json", &openapi3.Schema{Type: "string"}), + openapi3.NewSchemaRef("https://geojson.org/schema/MultiLineString.json", &openapi3.Schema{Type: "string"}), + openapi3.NewSchemaRef("https://geojson.org/schema/MultiPolygon.json", &openapi3.Schema{Type: "string"}), + }, + }, + }, + "properties": { + Value: &openapi3.Schema{ + Type: "object", + }, + }, + "bbox": { + Value: &openapi3.Schema{ + Type: "array", + MinItems: 4, + MaxItems: openapi3.Uint64Ptr(4), + Items: openapi3.NewSchemaRef("", openapi3.NewFloat64Schema().WithMin(-180).WithMax(180)), + }, + }, + }, + Example: getFeatureExample(), +} + // GetOpenAPIContent returns a Swagger OpenAPI structure func GetOpenAPIContent(urlBase string) *openapi3.Swagger { @@ -53,6 +116,16 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { AllowEmptyValue: false, }, } + paramFeatureID := openapi3.ParameterRef{ + Value: &openapi3.Parameter{ + Name: "featureId", + Description: "Id of feature in collection to retrieve data for.", + In: "path", + Required: true, + Schema: &openapi3.SchemaRef{Value: openapi3.NewStringSchema()}, + AllowEmptyValue: false, + }, + } paramBbox := openapi3.ParameterRef{ Value: &openapi3.Parameter{ Name: "bbox", @@ -365,6 +438,36 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { }, }, }, + Post: &openapi3.Operation{ + OperationID: "createCollectionFeature", + Parameters: openapi3.Parameters{ + ¶mCollectionID, + ¶mFeatureID, + // TODO keep it for the next evolution + // ¶mCrs, + }, + RequestBody: &openapi3.RequestBodyRef{ + Value: &openapi3.RequestBody{ + Description: "Add a new feature", + Required: true, + Content: openapi3.NewContentWithJSONSchema(&FeatureSchema), + }, + }, + Responses: openapi3.Responses{ + "201": &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: "Empty body with location header", + Headers: map[string]*openapi3.HeaderRef{ + "location": { + Value: &openapi3.Header{ + Description: "Contains a link to access to the new feature data", + }, + }, + }, + }, + }, + }, + }, }, apiBase + "collections/{collectionId}/schema": &openapi3.PathItem{ Summary: "Feature schema for collection", @@ -399,16 +502,7 @@ func GetOpenAPIContent(urlBase string) *openapi3.Swagger { OperationID: "getCollectionFeature", Parameters: openapi3.Parameters{ ¶mCollectionID, - &openapi3.ParameterRef{ - Value: &openapi3.Parameter{ - Name: "featureId", - Description: "Id of feature in collection to retrieve data for.", - In: "path", - Required: true, - Schema: &openapi3.SchemaRef{Value: openapi3.NewStringSchema()}, - AllowEmptyValue: false, - }, - }, + ¶mFeatureID, ¶mProperties, ¶mTransform, ¶mCrs, diff --git a/internal/service/handler_post_test.go b/internal/service/handler_post_test.go index 64c41a33..044fb935 100644 --- a/internal/service/handler_post_test.go +++ b/internal/service/handler_post_test.go @@ -42,11 +42,23 @@ func TestApiContainsCollectionSchemas(t *testing.T) { util.Assert(t, path != nil, "schema path exists") util.Equals(t, "Provides access to data representation (schema) for any features in specified collection", path.Description, "schema path present") util.Equals(t, "getCollectionSchema", path.Get.OperationID, "schema path get present") - util.Equals(t, 2, len(path.Get.Parameters), "schema path get present") + util.Equals(t, 2, len(path.Get.Parameters), "# path") util.Assert(t, path.Get.Parameters.GetByInAndName("path", "collectionId") != nil, "collectionId path parameter exists") util.Assert(t, path.Get.Parameters.GetByInAndName("query", "type") != nil, "type query parameter exists") } +// checks swagger api contains method PATCH for updating a feaure from a specified collection +func TestApiContainsMethodPostFeature(t *testing.T) { + resp := hTest.DoRequest(t, "/api") + body, _ := ioutil.ReadAll(resp.Body) + + var v openapi3.Swagger + err := json.Unmarshal(body, &v) + util.Assert(t, err == nil, fmt.Sprintf("%v", err)) + + util.Equals(t, "createCollectionFeature", v.Paths.Find("/collections/{collectionId}/items").Post.OperationID, "method POST present") +} + // checks collection schema contains valid data description func TestGetCollectionCreateSchema(t *testing.T) { path := "/collections/mock_a/schema?type=create"