Merge pull request #23 from i-dot-ai:feat/search-debates-by-title-recency

return most recent debates in search_debate_titles

Author: MarkDunne

parliament_mcp/mcp_server/api.py

@@ -107,15 +107,14 @@ async def search_debate_titles(
     Only returns the debate ID, title, and date, not the content of the debate.
     Useful for finding relevant debates, but must be used in conjunction with search_contributions to get the full text of the debate.
 
-    Either query or date range (or both) must be provided. House is optional.
-    If only date_from is provided, returns debates from that date onwards.
-    If only date_to is provided, returns debates up to and including that date.
+    Use date_from and date_to to search for debates in a specific date range.
+    Use query to search for debates with a keyword or phrase in the title.
 
-    Returns a list of debate details (ID, title, date) ranked by relevancy.
+    Returns a list of debate details (ID, title, date) in order the most recent first.
 
     Common use case for this function:
-    - Provide a query to search for debates on a specific topic for all time
-    - Provide the date for date_from and date_to to search for all debates on a specific date
+    - Provide a query to search for debates on a specific topic
+    - Provide the date for date_from and date_to to search for debates within a specific date range
     - Provide a query and date range to search for debates on a specific topic in a specific date range
 
     Args:

parliament_mcp/mcp_server/members.py

@@ -52,7 +52,7 @@ async def get_election_results(
         raise ValueError(msg)
 
 
-# @log_tool_call
+@log_tool_call
 async def search_members(
     Name: str | None = Field(None, description="Member name"),
     PartyId: int | None = Field(None, description="Party ID"),

parliament_mcp/mcp_server/qdrant_query_handler.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from datetime import datetime
 from typing import Any, Literal
 
@@ -53,6 +54,46 @@ def build_filters(conditions: list[FieldCondition | None]) -> Filter | None:
     return Filter(must=valid_conditions)
 
 
+class DebateCollection:
+    """Collection of debates and their contributions.
+    Used to track the contributions for each debate and return the substantial debates.
+    """
+
+    def __init__(self):
+        self._debates = defaultdict(lambda: {"contribution_ids": set(), "info": None})
+
+    def add_contribution(self, contribution):
+        debate_id = contribution.get("DebateSectionExtId")
+        contribution_id = contribution.get("ContributionExtId")
+        debate = self._debates[debate_id]
+        new_data = contribution_id not in debate["contribution_ids"]
+        debate["contribution_ids"].add(contribution_id)
+        if debate["info"] is None:
+            debate["info"] = {
+                "debate_id": debate_id,
+                "title": contribution.get("DebateSection"),
+                "date": contribution.get("SittingDate"),
+                "house": contribution.get("House"),
+                "debate_parents": contribution.get("debate_parents", []),
+                "debate_url": contribution.get("debate_url"),
+            }
+        return new_data
+
+    def get_substantial_debates(self):
+        return [
+            debate["info"]
+            for debate in self._debates.values()
+            if len(debate["contribution_ids"]) >= MINIMUM_DEBATE_HITS
+        ]
+
+    def get_substantial_debate_ids(self):
+        return [
+            debate_id
+            for debate_id, debate in self._debates.items()
+            if len(debate["contribution_ids"]) >= MINIMUM_DEBATE_HITS
+        ]
+
+
 class QdrantQueryHandler:
     def __init__(
         self, qdrant_client: AsyncQdrantClient, openai_client: AsyncAzureOpenAI, settings: ParliamentMCPSettings
@@ -82,7 +123,7 @@ async def search_debate_titles(
         date_from: str | None = None,
         date_to: str | None = None,
         house: str | None = None,
-        max_results: int = 100,
+        max_results: int = 50,
     ) -> list[dict]:
         """
         Search debate titles with optional filters.
@@ -116,34 +157,34 @@ async def search_debate_titles(
 
         query_filter = build_filters(filter_conditions)
 
-        query_response = await self.qdrant_client.query_points_groups(
-            collection_name=self.settings.HANSARD_CONTRIBUTIONS_COLLECTION,
-            query_filter=query_filter,
-            limit=max_results,
-            with_payload=True,
-            group_by="DebateSectionExtId",
-            group_size=MINIMUM_DEBATE_HITS,
-        )
+        debates = DebateCollection()
 
-        results = []
-        for group in query_response.groups:
-            first_hit = group.hits[0].payload
-            # If there are less than 2 hits, the debate is not relevant
-            if len(group.hits) < MINIMUM_DEBATE_HITS:
-                continue
-
-            debate = {
-                "debate_id": first_hit.get("DebateSectionExtId"),
-                "title": first_hit.get("DebateSection"),
-                "date": first_hit.get("SittingDate"),
-                "house": first_hit.get("House"),
-                "debate_parents": first_hit.get("debate_parents", []),
-                "debate_url": first_hit.get("debate_url"),
-            }
+        while len(substantial_ids := debates.get_substantial_debate_ids()) < max_results:
+            # Filter out already found substantial debates
+            if substantial_ids:
+                query_filter.must_not = [
+                    FieldCondition(key="DebateSectionExtId", match=models.MatchAny(any=substantial_ids)),
+                ]
+
+            contributions, _ = await self.qdrant_client.scroll(
+                collection_name=self.settings.HANSARD_CONTRIBUTIONS_COLLECTION,
+                scroll_filter=query_filter,
+                limit=1000,
+                with_payload=True,
+                order_by={"key": "SittingDate", "direction": "desc"},
+            )
 
-            results.append(debate)
+            if not contributions:
+                break
 
-        return results
+            new_data_available = False
+            for result in contributions:
+                new_data_available |= debates.add_contribution(result.payload)
+
+            if not new_data_available:
+                break
+
+        return debates.get_substantial_debates()[:max_results]
 
     async def search_hansard_contributions(
         self,
@@ -219,7 +260,7 @@ async def search_hansard_contributions(
             query_response = query_response.points
         else:
             # If no query, use scroll to get results with filters only
-            query_response, offset = await self.qdrant_client.scroll(
+            query_response, _ = await self.qdrant_client.scroll(
                 collection_name=self.settings.HANSARD_CONTRIBUTIONS_COLLECTION,
                 scroll_filter=query_filter,
                 limit=max_results,
@@ -429,13 +470,13 @@ async def search_parliamentary_questions(
 
         else:
             # If no query, use scroll to get results with filters only
-            query_response, offset = await self.qdrant_client.scroll(
+            query_response, _ = await self.qdrant_client.scroll(
                 collection_name=self.settings.PARLIAMENTARY_QUESTIONS_COLLECTION,
                 scroll_filter=query_filter,
                 limit=max_results,
                 with_payload=True,
                 order_by={
-                    "key": "created_at",
+                    "key": "id",
                     "direction": "desc",
                 },
             )

parliament_mcp/qdrant_helpers.py

@@ -182,7 +182,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.DatetimeIndexParams(
             type=models.DatetimeIndexType.DATETIME,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -191,7 +191,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.DatetimeIndexParams(
             type=models.DatetimeIndexType.DATETIME,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -200,7 +200,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.KeywordIndexParams(
             type=models.KeywordIndexType.KEYWORD,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -209,7 +209,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.IntegerIndexParams(
             type=models.IntegerIndexType.INTEGER,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -218,7 +218,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.KeywordIndexParams(
             type=models.KeywordIndexType.KEYWORD,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -234,7 +234,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
             stopwords="english",
             stemmer=models.SnowballParams(type=models.Snowball.SNOWBALL, language=models.SnowballLanguage.ENGLISH),
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -245,13 +245,6 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
             lookup=True,
             range=True,
         ),
-        wait=False,
-    )
-
-    await client.create_payload_index(
-        collection_name=settings.PARLIAMENTARY_QUESTIONS_COLLECTION,
-        field_name="created_at",
-        field_schema=models.DatetimeIndexParams(type=models.DatetimeIndexType.DATETIME),
         wait=True,
     )
 
@@ -263,7 +256,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.DatetimeIndexParams(
             type=models.DatetimeIndexType.DATETIME,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -272,7 +265,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.KeywordIndexParams(
             type=models.KeywordIndexType.KEYWORD,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -283,7 +276,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
             lookup=True,
             range=True,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -292,7 +285,7 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
         field_schema=models.KeywordIndexParams(
             type=models.KeywordIndexType.KEYWORD,
         ),
-        wait=False,
+        wait=True,
     )
 
     await client.create_payload_index(
@@ -308,5 +301,5 @@ async def create_collection_indicies(client: AsyncQdrantClient, settings: Parlia
             stopwords="english",
             stemmer=models.SnowballParams(type=models.Snowball.SNOWBALL, language=models.SnowballLanguage.ENGLISH),
         ),
-        wait=False,
+        wait=True,
     )