diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index 77069926..57c64ca4 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,6 +363,11 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } + if ( data.aggregations?.count_by_place ) { + response.aggregations = { + count_by_place: data.aggregations.count_by_place + }; + } response.page = Number( req.elastic_query.page ); response.per_page = Number( req.elastic_query.per_page ); response.results = obs; @@ -411,6 +416,71 @@ ObservationsController.speciesCounts = async req => { return TaxaController.speciesCountsResponse( req, leafCounts ); }; +ObservationsController.placeCountsCacheWrapper = async req => ( + ObservationsController.methodCacheWrapper( req, + ObservationsController.placeCounts, + "ObservationsController.placeCounts" ) +); + +ObservationsController.placeCounts = async ( req, options = {} ) => { + const page = parseInt( req.query.page || 1, 10 ); + const perPage = parseInt( req.query.per_page || 30, 10 ); + const offset = ( page - 1 ) * perPage; + const orderAscDesc = req.query.order === "asc" ? "asc" : "desc"; + const order = req.query.order_by === "id" ? { _key: orderAscDesc } : { _count: orderAscDesc }; + const idsToFilter = util.paramArray( req.query.count_place_id ); + const shortenedIdsToFilter = _.isArray( idsToFilter ) + ? req.query.count_place_id.slice( 0, 1000 ) + : []; + const countByPlaceTermsAgg = { + count_by_place: { + terms: { + field: "place_ids.keyword", + size: 1000, + // according to Elastic docs, _count "asc" can return inaccurate results due to sharding + order + } + } + }; + if ( !_.isEmpty( shortenedIdsToFilter ) ) { + countByPlaceTermsAgg.count_by_place.terms.include = idsToFilter; + } + req.query.aggs = countByPlaceTermsAgg; + req.query.per_page = 0; + + const data = await ObservationsController.resultsForRequest( req, options ); + const buckets = data.aggregations?.count_by_place?.buckets || []; + + const places = await Promise.all( + buckets.map( bucket => Place.findByID( bucket.key, { + fields: ["id", "name", "display_name"] + } ) ) + ); + + const results = places + .map( ( place, i ) => { + if ( !place ) return null; + return { + count: buckets[i].doc_count, + place: { + id: place.id, + name: place.name, + display_name: place.display_name + } + }; + } ) + .filter( Boolean ); + + const paginated = results.slice( offset, offset + perPage ); + + return { + total_results: results.length, + page, + per_page: perPage, + results: paginated + }; +}; + ObservationsController.taxa = async req => { if ( !req.query.user_id ) { throw new Error( 422 ); diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index b51fd2ad..c97148e4 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,6 +298,9 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); + dfault( "get", "/v1/observations/place_counts", ObservationsController.placeCountsCacheWrapper, { + setTTL: true + } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { setTTL: true } ); diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index e233caac..6c594882 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1046,6 +1046,36 @@ paths: description: Unexpected error schema: $ref: "#/definitions/Error" + /observations/place_counts: + get: + summary: Observation Place Counts + description: | + Given zero to many of the following parameters, returns the number of observations matching + the search criteria grouped by place. Each result includes the place and the count of + associated observations. This endpoint works similarly to `/observations`, but instead of + returning individual observations, it returns aggregated results per place. A + maximum of 1000 place-count entries can be aggregated for a given query. Use the parameter + `count_place_id` to specify the places for which to return counts. + parameters: + <%- include( "_observation_search_params_v1.yml.ejs" ) %> + - $ref: "#/parameters/count_place_id" + - $ref: "#/parameters/page" + - $ref: "#/parameters/per_page" + - $ref: "#/parameters/order" + - $ref: "#/parameters/place_counts_order_by" + - $ref: "#/parameters/only_id" + tags: + - Observations + responses: + 200: + description: | + Returns an object with metadata and an array of taxa + schema: + $ref: "#/definitions/PlacesCountsResponse" + default: + description: Unexpected error + schema: + $ref: "#/definitions/Error" /observations/popular_field_values: get: summary: Observation Popular Field Values @@ -2168,6 +2198,16 @@ parameters: type: boolean in: query description: Captive or cultivated observations + count_place_id: + type: array + items: + type: integer + in: query + description: | + Places where from aggregate observation counts are returned. + If more than 1,000 id's are specified, the first 1,000 are used. + If `count_place_id` is not specified, results will contain any places included in observations returned from your query; + note that therefore it may include other places you have not included in the `place_id` parameter if specified endemic: name: endemic type: boolean @@ -2826,6 +2866,15 @@ parameters: in: query description: | Number of results to return in a `page`. The maximum value is 500 + place_counts_order_by: + name: order_by + type: string + in: query + description: Sort field. We strongly discourage count when combined with order = 'asc' as results may be inaccurate. + default: count + enum: + - count + - id q: name: q type: string @@ -4699,6 +4748,21 @@ definitions: type: integer taxon: $ref: "#/definitions/ShowTaxon" + PlacesCountsResponse: + allOf: + - $ref: "#/definitions/BaseResponse" + - required: + - results + properties: + results: + type: array + items: + type: object + properties: + count: + type: integer + place: + $ref: "#/definitions/CorePlace" PlacesResponse: allOf: - $ref: "#/definitions/BaseResponse" diff --git a/schema/fixtures.js b/schema/fixtures.js index d75370e9..0bfe3787 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -99,6 +99,7 @@ "name": "United States", "slug": "united-states", "display_name_autocomplete": "United States", + "display_name": "United States", "location": "48.8907012939,-116.9820022583", "admin_level": 0, "bbox_area": 5500, @@ -252,6 +253,20 @@ ] ] } + }, + { + "id": 2025101508, + "name": "Testing Place Counts Island" + }, + { + "id": 2025101509, + "name": "Forest In Testing Place Counts Island", + "ancestor_place_ids": [2025101508, 2025101509] + }, + { + "id": 2025101510, + "name": "Lake In Testing Place Counts Island", + "ancestor_place_ids": [2025101508, 2025101510] } ] }, @@ -1222,6 +1237,24 @@ } ], "sounds_count": 1 + }, + { + "id": 2025101501, + "uuid": "testing-obsv-places-count1", + "place_ids": [2025101508, 2025101509], + "private_place_ids": [2025101508, 2025101509] + }, + { + "id": 2025101502, + "uuid": "testing-obsv-places-count2", + "place_ids": [2025101508, 2025101510], + "private_place_ids": [2025101508, 2025101510] + }, + { + "id": 2025101503, + "uuid": "testing-obsv-places-count3", + "place_ids": [2025101508, 2025101510], + "private_place_ids": [2025101508, 2025101510] } ] }, @@ -2763,6 +2796,18 @@ { "id": 2025012202, "uuid": "537d9290-df3c-404f-a26d-ffe07156908c" + }, + { + "id": 2025101501, + "uuid": "testing-obsv-places-count1" + }, + { + "id": 2025101502, + "uuid": "testing-obsv-places-count2" + }, + { + "id": 2025101503, + "uuid": "testing-obsv-places-count3" } ], "observation_photos": [ diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 7e03ad8a..e353e4f8 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1409,6 +1409,67 @@ describe( "Observations", ( ) => { } ); } ); + describe( "place_counts", ( ) => { + it( "returns JSON", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ) + .expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + + it( "sorts by count desc by default", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); + } ).expect( 200, done ); + } ); + + it( "can sort by count asc", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?order=asc&order_by=count" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); + } ).expect( 200, done ); + } ); + + it( "supports pagination", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=count&per_page=1&page=2" ).expect( res => { + expect( res.body.page ).to.eq( 2 ); + expect( res.body.per_page ).to.eq( 1 ); + } ).expect( 200, done ); + } ); + + it( "returns results places counts with expected fields", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts" ).expect( res => { + const result = res.body.results[0]; + expect( result ).to.have.property( "count" ); + expect( result.place ).to.have.property( "id" ); + expect( result.place ).to.have.property( "name" ); + expect( result.place ).to.have.property( "display_name" ); + } ).expect( 200, done ); + } ); + + it( "returns counts from all places filtered in observations when place_id is specified", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508" ).expect( res => { + expect( res.body.results.length ).to.be.eq( 3 ); + expect( res.body.results[0].place.id ).to.be.eq( 2025101508 ); + expect( res.body.results[0].count ).to.be.eq( 3 ); + expect( res.body.results[1].place.id ).to.be.eq( 2025101510 ); + expect( res.body.results[1].count ).to.be.eq( 2 ); + expect( res.body.results[2].place.id ).to.be.eq( 2025101509 ); + expect( res.body.results[2].count ).to.be.eq( 1 ); + } ).expect( 200, done ); + } ); + + it( "returns counts only from count_place_id when both it and place_id are specified", function ( done ) { + request( this.app ).get( "/v1/observations/place_counts?place_id=2025101508&count_place_id=2025101510%2C2025101509&order=asc&order_by=id" ).expect( res => { + expect( res.body.results.length ).to.be.eq( 2 ); + expect( res.body.results[0].place.id ).to.be.eq( 2025101509 ); + expect( res.body.results[0].count ).to.be.eq( 1 ); + expect( res.body.results[1].place.id ).to.be.eq( 2025101510 ); + expect( res.body.results[1].count ).to.be.eq( 2 ); + } ).expect( 200, done ); + } ); + } ); + describe( "iconic_taxa_counts", ( ) => { it( "returns json", function ( done ) { request( this.app ).get( "/v1/observations/iconic_taxa_counts" )