cache related tables, find recent surveys

Author: austinorr

agolutils/arcgis/survey123.py

@@ -7,7 +7,7 @@
 import pandas
 
 
-from agolutils.utils import make_path
+from agolutils.utils import make_path, yesterday, tomorrow
 from agolutils.context import write_context
 from .utils import get_content
 
@@ -89,7 +89,7 @@ def download_all_attachments(
 
 def get_layer_records_by_objectid(layer, oids: List[int]):
     object_ids = ",".join(map(str, oids))
-    q = layer.query(object_ids=object_ids)
+    q = layer.query(object_ids=object_ids, return_geometry=False)
     df = q.sdf
     return json.loads(df.to_json(orient="records"))
 
@@ -99,13 +99,27 @@ def get_layer_by_prop(service, prop, equals):
     return next(filter(fxn, service.layers + service.tables))
 
 
-def get_related_records(service, layer, oid, context_dir, download_attachments=True):
-    relates = {}
-
+def get_related_tables(service, layer):
+    tables = []
     for relationship in layer.properties.relationships:
         rel_id = relationship["id"]
         tableid = relationship["relatedTableId"]
         table = get_layer_by_prop(service, "id", tableid)
+        tables.append({"rel_id": rel_id, "table": table})
+    return tables
+
+
+def get_related_records(
+    layer, oid, context_dir, service=None, tables=None, download_attachments=True
+):
+    relates = {}
+
+    if tables is None:
+        tables = get_related_tables(service, layer)
+
+    for rel in tables:
+        rel_id = rel["rel_id"]
+        table = rel["table"]
         name = table.properties.name
 
         related_record_groups = layer.query_related_records(oid, rel_id).get(
@@ -138,7 +152,9 @@ def get_related_records(service, layer, oid, context_dir, download_attachments=T
                     ]
                 )
 
-        rel_df = table.query(object_ids=",".join(map(str, rel_oids))).sdf
+        rel_df = table.query(
+            object_ids=",".join(map(str, rel_oids)), return_geometry=False
+        ).sdf
         if files:
             rel_df = rel_df.merge(pandas.DataFrame(files), on="objectid")
 
@@ -166,12 +182,13 @@ def slim_layer_properties(props: dict):
 
 
 def get_contexts(
-    service,
     layer,
     oids,
     context_file_pattern=None,
     download_attachments=True,
     get_relates=True,
+    tables=None,
+    service=None,
 ):
     records = get_layer_records_by_objectid(layer, oids)
     if context_file_pattern is None:
@@ -192,11 +209,12 @@ def get_contexts(
 
         if get_relates:
             context["relates"] = get_related_records(
-                service,
                 layer,
                 oid,
-                download_attachments=download_attachments,
                 context_dir=outdir,
+                tables=tables,
+                service=service,
+                download_attachments=download_attachments,
             )
 
         if download_attachments and layer.properties.hasAttachments:
@@ -206,7 +224,7 @@ def get_contexts(
                     layer,
                     oid=oid,
                     attachment_id=att["id"],
-                    save_path=outdir,
+                    save_path=str(outdir),
                     filename_prefix=context["_layer_name"] + "-",
                 )
 
@@ -218,22 +236,74 @@ def get_contexts(
     return contexts
 
 
+def get_recent(layer, related_tables=None, start_date=None, end_date=None):
+    if related_tables is None:
+        related_tables = []
+    if start_date is None:
+        start_date = yesterday()
+    if end_date is None:
+        end_date = tomorrow()
+
+    time_query = (
+        f"EditDate >= TIMESTAMP '{start_date}' and EditDate <= TIMESTAMP '{end_date}'"
+    )
+
+    globalids = None
+
+    if related_tables:
+        dfs = []
+        for t in related_tables:
+            sdf = (
+                t["table"]
+                .query(time_query, out_fields=["parentglobalid"], return_geometry=False)
+                .sdf
+            )
+            if not sdf.empty:
+                dfs.extend(sdf["parentglobalid"].unique())
+        recent_surveys = set(dfs)
+        globalids = ", ".join([f"'{x}'" for x in recent_surveys])
+
+    q = f"({time_query})"
+    if globalids:
+        q += f"or (globalid in ({globalids}))"
+
+    sdf = layer.query(q, out_fields=["objectid"], return_geometry=False).sdf
+
+    return [] if sdf.empty else sdf["objectid"].to_list()
+
+
 class Survey123Service:
     def __init__(self, item: arcgis.gis.Item):
         self.service = item
 
         self._survey_layer = None
         self._survey_properties = None
-        self._tables = None
+        self._related_tables = None
 
     @property
     def survey_layer(self):
         if self._survey_layer is None:
             self._survey_layer = self.service.layers[0]
         return self._survey_layer
 
+    @property
+    def related_tables(self):
+        if self._related_tables is None:
+            self._related_tables = get_related_tables(self.service, self.survey_layer)
+        return self._related_tables
+
+    def get_recent(self, start_date=None, end_date=None):
+        return get_recent(
+            self.survey_layer,
+            related_tables=self.related_tables,
+            start_date=start_date,
+            end_date=end_date,
+        )
+
     def get_related_records(self, oid, context_dir):
-        return get_related_records(self.service, self.survey_layer, oid, context_dir)
+        return get_related_records(
+            self.survey_layer, oid, context_dir, tables=self.related_tables
+        )
 
     def get_contexts(
         self,
@@ -243,9 +313,10 @@ def get_contexts(
         get_relates=True,
     ):
         return get_contexts(
-            self.service,
             self.survey_layer,
             oids,
+            tables=self.related_tables,
+            service=self.service,
             context_file_pattern=context_file_pattern,
             download_attachments=download_attachments,
             get_relates=get_relates,

agolutils/utils.py

@@ -92,3 +92,7 @@ def format_date(timestamp, tz_string=None, fmt=None):
 
     else:
         return utc_dt
+
+
+tomorrow = lambda : (datetime.datetime.utcnow() + datetime.timedelta(1)).strftime("%Y-%m-%d")
+yesterday = lambda : (datetime.datetime.utcnow() - datetime.timedelta(1)).strftime("%Y-%m-%d")