From 7547f8a9d26693f23e779c501eca1b44c011fe2e Mon Sep 17 00:00:00 2001 From: Simon Prickett Date: Fri, 6 Oct 2023 14:06:04 +0100 Subject: [PATCH 1/2] Updated data loader for redis-py GEOSHAPE support. --- data_loader.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/data_loader.py b/data_loader.py index 2fe1af7..85d3782 100644 --- a/data_loader.py +++ b/data_loader.py @@ -1,12 +1,17 @@ from dotenv import load_dotenv from pathlib import Path from time import sleep +from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.field import TextField, TagField, GeoShapeField import argparse import json import os import redis +WEATHER_KEY_PREFIX = "region" +WEATHER_INDEX_NAME = "idx:regions" + load_dotenv() # Parse arguments and make sure we were invoked correctly. @@ -27,7 +32,7 @@ # Drop any existing index. try: print("Checking for previous index and dropping if found.") - redis_client.ft("idx:regions").dropindex(delete_documents = False) + redis_client.ft(WEATHER_INDEX_NAME).dropindex(delete_documents = False) print("Dropped old search index.") except redis.exceptions.ResponseError as e: # Dropping an index that doesn't exist throws an exception @@ -42,7 +47,20 @@ # Create a new index. print("Creating index.") -redis_client.execute_command("FT.CREATE", "idx:regions", "ON", "JSON", "PREFIX", "1", "region:", "SCHEMA", "$.name", "AS", "name", "TAG", "$.boundaries", "AS", "boundaries", "GEOSHAPE", "SPHERICAL", "$.forecast.wind", "AS", "WIND", "TEXT", "$.forecast.sea", "AS", "sea", "TEXT", "$.forecast.weather", "AS", "weather", "TEXT", "$.forecast.visibility", "AS", "visibility", "TEXT") +redis_client.ft(WEATHER_INDEX_NAME).create_index( + [ + TagField("$.name", as_name = "name"), + GeoShapeField("$.boundaries", GeoShapeField.SPHERICAL, as_name = "boundaries"), + TextField("$.forecast.wind", as_name = "wind"), + TextField("$.forecast.sea", as_name = "sea"), + TextField("$.forecast.weather", as_name = "weather"), + TextField("$.forecast.visibility", as_name = "visibility") + ], + definition = IndexDefinition( + index_type = IndexType.JSON, + prefix = [ f"{WEATHER_KEY_PREFIX}:" ] + ) +) # Load the shipping forecast regional data from the JSON file. num_loaded = 0 @@ -51,7 +69,7 @@ file_data = json.load(input_file) for region in file_data["regions"]: - redis_key = f"region:{region['name'].replace(' ', '_').lower()}" + redis_key = f"{WEATHER_KEY_PREFIX}:{region['name'].replace(' ', '_').lower()}" redis_client.json().set(redis_key, "$", region) num_loaded += 1 print(f"Stored {redis_key} ({region['name']})") From 67a4833f4a76c0c8bed8f62eb98c02a40d1ba2c5 Mon Sep 17 00:00:00 2001 From: Simon Prickett Date: Mon, 9 Oct 2023 18:23:52 +0100 Subject: [PATCH 2/2] Updated README for better FT.CREATE support. --- README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 356e909..74745ef 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Polygon search example in action](screenshots/polyweather.gif) -Watch the recording of our Polygon Search live stream video on YouTube [here](https://www.youtube.com/watch?v=CegTSglMUks). Note that the code has been tidied up a little since this was recorded -- it no longer uses redis-py's `execute_command` when performing search queries: it uses the more idiomatic interface instead. +Watch the recording of our Polygon Search live stream video on YouTube [here](https://www.youtube.com/watch?v=CegTSglMUks). Note that the code has been tidied up a little since this was recorded -- redis-py was updated so the code no longer uses `execute_command` when performing search queries and creating the search index: it uses the more idiomatic interface instead. ## Introduction @@ -182,15 +182,19 @@ with open (args.data_file_name, "r") as input_file: The data loader script also creates the search index. It first deletes any previous index definition, then runs the [`FT.CREATE`](https://redis.io/commands/ft.create/) command: ```python -redis_client.execute_command( - "FT.CREATE", "idx:regions", "ON", "JSON", "PREFIX", "1", "region:", - "SCHEMA", - "$.name", "AS", "name", "TAG", - "$.boundaries", "AS", "boundaries", "GEOSHAPE", "SPHERICAL", - "$.forecast.wind", "AS", "WIND", "TEXT", - "$.forecast.sea", "AS", "sea", "TEXT", - "$.forecast.weather", "AS", "weather", "TEXT", - "$.forecast.visibility", "AS", "visibility", "TEXT" +redis_client.ft(WEATHER_INDEX_NAME).create_index( + [ + TagField("$.name", as_name = "name"), + GeoShapeField("$.boundaries", GeoShapeField.SPHERICAL, as_name = "boundaries"), + TextField("$.forecast.wind", as_name = "wind"), + TextField("$.forecast.sea", as_name = "sea"), + TextField("$.forecast.weather", as_name = "weather"), + TextField("$.forecast.visibility", as_name = "visibility") + ], + definition = IndexDefinition( + index_type = IndexType.JSON, + prefix = [ f"{WEATHER_KEY_PREFIX}:" ] + ) ) ``` @@ -207,8 +211,6 @@ The front end doesn't currently allow for searching by anything other than `boun Note that the order of creating the index and loading the documents doesn't matter. In this example, we're creating the index first but it could be done the other way around. The Search capability of Redis Stack will index documents for us from the moment the index is created, then track changes in the indexed area of the keyspace. It automatically adds, updates and deletes index entries as changes occur to tracked documents. -Note also that we're using the generic `execute_command` function here as redis-py doesn't yet support the `GEOSHAPE` syntax in its more idiomatic `ft("index name").create_index` implementation. I'll revisit this code when this changes. - ### Serving a Map and Defining the Search Polygon The front end uses [Leaflet maps](https://leafletjs.com/) with the [OpenStreetMap](https://www.openstreetmap.org/) tile layer. It's beyond the scope of this document to explain how this works - if you're curious check out Leaflet's [quick start](https://leafletjs.com/examples/quick-start/). At a high level, we load the JavaScript and configure a map to appear in a given `div` on the page by providing the ID of the `div`, a lat/long centre point for the map and an initial zoom level: