feat: support GitHub issues (#433)

* feat: support GitHub issues

* Better support for JSON fields

CHANGELOG.rst

@@ -5,6 +5,8 @@ Changelog
 Next
 ====
 
+- Add support for GitHub issues (#433)
+
 Version 1.2.16 - 2024-02-22
 ===========================
 

src/shillelagh/adapters/api/github.py

@@ -1,6 +1,7 @@
 """
 An adapter for GitHub.
 """
+import json
 import logging
 import urllib.parse
 from dataclasses import dataclass
@@ -20,6 +21,18 @@
 PAGE_SIZE = 100
 
 
+class JSONString(Field[Any, str]):
+    """
+    A field to handle JSON values.
+    """
+
+    type = "TEXT"
+    db_api_type = "STRING"
+
+    def parse(self, value: Any) -> Optional[str]:
+        return value if value is None else json.dumps(value)
+
+
 @dataclass
 class Column:
     """
@@ -63,6 +76,26 @@ class Column:
             Column("closed_at", "closed_at", StringDateTime()),
             Column("merged_at", "merged_at", StringDateTime()),
         ],
+        "issues": [
+            Column("url", "html_url", String()),
+            Column("id", "id", Integer()),
+            Column("number", "number", Integer(filters=[Equal])),
+            Column("state", "state", String(filters=[Equal]), Equal("all")),
+            Column("title", "title", String()),
+            Column("userid", "user.id", Integer()),
+            Column("username", "user.login", String()),
+            Column("draft", "draft", Boolean()),
+            Column("locked", "locked", Boolean()),
+            Column("comments", "comments", Integer()),
+            Column("created_at", "created_at", StringDateTime()),
+            Column("updated_at", "updated_at", StringDateTime()),
+            Column("closed_at", "closed_at", StringDateTime()),
+            Column("body", "body", String()),
+            Column("author_association", "author_association", String()),
+            Column("labels", "labels[*].name", JSONString()),
+            Column("assignees", "assignees[*].login", JSONString()),
+            Column("reactions", "reactions", JSONString()),
+        ],
     },
 }
 
@@ -177,7 +210,7 @@ def _get_single_resource(
         payload = response.json()
 
         row = {
-            column.name: jsonpath.findall(column.json_path, payload)[0]
+            column.name: get_value(column, payload)
             for column in TABLES[self.base][self.resource]
         }
         row["rowid"] = 0
@@ -231,7 +264,7 @@ def _get_multiple_resources(
                     break
 
                 row = {
-                    column.name: jsonpath.findall(column.json_path, resource)[0]
+                    column.name: get_value(column, resource)
                     for column in TABLES[self.base][self.resource]
                 }
                 row["rowid"] = rowid
@@ -240,3 +273,18 @@ def _get_multiple_resources(
                 rowid += 1
 
             page += 1
+
+
+def get_value(column: Column, resource: Dict[str, Any]) -> Any:
+    """
+    Extract the value of a column from a resource.
+    """
+    values = jsonpath.findall(column.json_path, resource)
+
+    if isinstance(column.field, JSONString):
+        return values
+
+    try:
+        return values[0]
+    except IndexError:
+        return None

tests/adapters/api/github_test.py

@@ -14,7 +14,11 @@
 from shillelagh.exceptions import ProgrammingError
 from shillelagh.filters import Equal
 
-from ...fakes import github_response, github_single_response
+from ...fakes import (
+    github_issues_response,
+    github_pulls_response,
+    github_single_response,
+)
 
 
 def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None:
@@ -27,7 +31,7 @@ def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None:
     )
 
     page1_url = "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=100&page=1"
-    requests_mock.get(page1_url, json=github_response)
+    requests_mock.get(page1_url, json=github_pulls_response)
     page2_url = "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=100&page=2"
     requests_mock.get(page2_url, json=[])
 
@@ -206,11 +210,11 @@ def test_github_limit_offset(mocker: MockerFixture, requests_mock: Mocker) -> No
     page2_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=2"
     )
-    requests_mock.get(page2_url, json=github_response[:5])
+    requests_mock.get(page2_url, json=github_pulls_response[:5])
     page3_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=3"
     )
-    requests_mock.get(page3_url, json=github_response[5:])
+    requests_mock.get(page3_url, json=github_pulls_response[5:])
 
     connection = connect(":memory:")
     cursor = connection.cursor()
@@ -466,11 +470,11 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) ->
     page2_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=2"
     )
-    requests_mock.get(page2_url, json=github_response[:5])
+    requests_mock.get(page2_url, json=github_pulls_response[:5])
     page3_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=3"
     )
-    requests_mock.get(page3_url, json=github_response[5:])
+    requests_mock.get(page3_url, json=github_pulls_response[5:])
 
     adapter = GitHubAPI("repos", "apache", "superset", "pulls")
     rows = adapter._get_multiple_resources(  # pylint: disable=protected-access
@@ -560,3 +564,80 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) ->
             "rowid": 4,
         },
     ]
+
+
+def test_github_missing_field(mocker: MockerFixture, requests_mock: Mocker) -> None:
+    """
+    Test a request when the response is missing a field.
+
+    For example, some issues don't have the ``draft`` field in the response.
+    """
+    mocker.patch(
+        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        return_value=Session(),
+    )
+
+    page1_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=1"
+    requests_mock.get(page1_url, json=github_issues_response)
+    page2_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=2"
+    requests_mock.get(page2_url, json=[])
+
+    connection = connect(":memory:")
+    cursor = connection.cursor()
+
+    sql = """
+        SELECT draft FROM
+        "https://api.github.com/repos/apache/superset/issues"
+        LIMIT 10
+    """
+    data = list(cursor.execute(sql))
+    assert data == [
+        (False,),
+        (False,),
+        (None,),
+        (None,),
+        (False,),
+        (None,),
+        (False,),
+        (None,),
+        (False,),
+        (False,),
+    ]
+
+
+def test_github_json_field(mocker: MockerFixture, requests_mock: Mocker) -> None:
+    """
+    Test a request when the response has a JSON field.
+    """
+    mocker.patch(
+        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        return_value=Session(),
+    )
+
+    page1_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=1"
+    requests_mock.get(page1_url, json=github_issues_response)
+    page2_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=2"
+    requests_mock.get(page2_url, json=[])
+
+    connection = connect(":memory:")
+    cursor = connection.cursor()
+
+    sql = """
+        SELECT labels FROM
+        "https://api.github.com/repos/apache/superset/issues"
+        WHERE labels != '[]'
+        LIMIT 10
+    """
+    data = list(cursor.execute(sql))
+    assert data == [
+        ('["size/M", "dependencies:npm", "github_actions", "packages"]',),
+        ('["size/S"]',),
+        ('["size/M"]',),
+        ('["size/M", "api"]',),
+        ('["size/L", "api"]',),
+        ('["size/XS"]',),
+        ('["size/XS", "dependencies:npm"]',),
+        ('["size/S"]',),
+        ('["size/XS", "hold:review-after-release"]',),
+        ('["size/M", "review-checkpoint", "plugins"]',),
+    ]

tests/fakes/__init__.py

@@ -128,7 +128,9 @@ def delete_data(self, row_id: int) -> None:
     datasette_results = [tuple(row) for row in json.load(fp)]
 with open(os.path.join(dirname, "incidents.json"), encoding="utf-8") as fp:
     incidents = json.load(fp)
-with open(os.path.join(dirname, "github_response.json"), encoding="utf-8") as fp:
-    github_response = json.load(fp)
+with open(os.path.join(dirname, "github_pulls_response.json"), encoding="utf-8") as fp:
+    github_pulls_response = json.load(fp)
+with open(os.path.join(dirname, "github_issues_response.json"), encoding="utf-8") as fp:
+    github_issues_response = json.load(fp)
 with open(os.path.join(dirname, "github_single_response.json"), encoding="utf-8") as fp:
     github_single_response = json.load(fp)