feat: support GitHub issues

CHANGELOG.rst

@@ -5,6 +5,8 @@ Changelog
 Next
 ====
 
+- Add support for GitHub issues (#433)
+
 Version 1.2.16 - 2024-02-22
 ===========================
 

src/shillelagh/adapters/api/github.py

@@ -63,6 +63,22 @@ class Column:
             Column("closed_at", "closed_at", StringDateTime()),
             Column("merged_at", "merged_at", StringDateTime()),
         ],
+        "issues": [
+            Column("url", "html_url", String()),
+            Column("id", "id", Integer()),
+            Column("number", "number", Integer(filters=[Equal])),
+            Column("state", "state", String(filters=[Equal]), Equal("all")),
+            Column("title", "title", String()),
+            Column("userid", "user.id", Integer()),
+            Column("username", "user.login", String()),
+            Column("draft", "draft", Boolean()),
+            Column("locked", "locked", Boolean()),
+            Column("comments", "comments", Integer()),
+            Column("created_at", "created_at", StringDateTime()),
+            Column("updated_at", "updated_at", StringDateTime()),
+            Column("closed_at", "closed_at", StringDateTime()),
+            Column("body", "body", String()),
+        ],
     },
 }
 
@@ -177,7 +193,7 @@ def _get_single_resource(
         payload = response.json()
 
         row = {
-            column.name: jsonpath.findall(column.json_path, payload)[0]
+            column.name: get_path_or_none(payload, column.json_path)
             for column in TABLES[self.base][self.resource]
         }
         row["rowid"] = 0
@@ -231,7 +247,7 @@ def _get_multiple_resources(
                     break
 
                 row = {
-                    column.name: jsonpath.findall(column.json_path, resource)[0]
+                    column.name: get_path_or_none(resource, column.json_path)
                     for column in TABLES[self.base][self.resource]
                 }
                 row["rowid"] = rowid
@@ -240,3 +256,13 @@ def _get_multiple_resources(
                 rowid += 1
 
             page += 1
+
+
+def get_path_or_none(resource: Dict[str, Any], path: str) -> Optional[Any]:
+    """
+    Return the value at ``path`` in ``resource`` or ``None`` if it doesn't exist.
+    """
+    try:
+        return jsonpath.findall(path, resource)[0]
+    except IndexError:
+        return None

tests/adapters/api/github_test.py

@@ -14,7 +14,11 @@
 from shillelagh.exceptions import ProgrammingError
 from shillelagh.filters import Equal
 
-from ...fakes import github_response, github_single_response
+from ...fakes import (
+    github_issues_response,
+    github_pulls_response,
+    github_single_response,
+)
 
 
 def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None:
@@ -27,7 +31,7 @@ def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None:
     )
 
     page1_url = "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=100&page=1"
-    requests_mock.get(page1_url, json=github_response)
+    requests_mock.get(page1_url, json=github_pulls_response)
     page2_url = "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=100&page=2"
     requests_mock.get(page2_url, json=[])
 
@@ -206,11 +210,11 @@ def test_github_limit_offset(mocker: MockerFixture, requests_mock: Mocker) -> No
     page2_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=2"
     )
-    requests_mock.get(page2_url, json=github_response[:5])
+    requests_mock.get(page2_url, json=github_pulls_response[:5])
     page3_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=3"
     )
-    requests_mock.get(page3_url, json=github_response[5:])
+    requests_mock.get(page3_url, json=github_pulls_response[5:])
 
     connection = connect(":memory:")
     cursor = connection.cursor()
@@ -466,11 +470,11 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) ->
     page2_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=2"
     )
-    requests_mock.get(page2_url, json=github_response[:5])
+    requests_mock.get(page2_url, json=github_pulls_response[:5])
     page3_url = (
         "https://api.github.com/repos/apache/superset/pulls?state=all&per_page=5&page=3"
     )
-    requests_mock.get(page3_url, json=github_response[5:])
+    requests_mock.get(page3_url, json=github_pulls_response[5:])
 
     adapter = GitHubAPI("repos", "apache", "superset", "pulls")
     rows = adapter._get_multiple_resources(  # pylint: disable=protected-access
@@ -560,3 +564,42 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) ->
             "rowid": 4,
         },
     ]
+
+
+def test_github_missing_field(mocker: MockerFixture, requests_mock: Mocker) -> None:
+    """
+    Test a request when the response is missing a field.
+
+    For example, some issues don't have the ``draft`` field in the response.
+    """
+    mocker.patch(
+        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        return_value=Session(),
+    )
+
+    page1_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=1"
+    requests_mock.get(page1_url, json=github_issues_response)
+    page2_url = "https://api.github.com/repos/apache/superset/issues?state=all&per_page=100&page=2"
+    requests_mock.get(page2_url, json=[])
+
+    connection = connect(":memory:")
+    cursor = connection.cursor()
+
+    sql = """
+        SELECT draft FROM
+        "https://api.github.com/repos/apache/superset/issues"
+        LIMIT 10
+    """
+    data = list(cursor.execute(sql))
+    assert data == [
+        (False,),
+        (False,),
+        (None,),
+        (None,),
+        (False,),
+        (None,),
+        (False,),
+        (None,),
+        (False,),
+        (False,),
+    ]

tests/fakes/__init__.py

@@ -128,7 +128,9 @@ def delete_data(self, row_id: int) -> None:
     datasette_results = [tuple(row) for row in json.load(fp)]
 with open(os.path.join(dirname, "incidents.json"), encoding="utf-8") as fp:
     incidents = json.load(fp)
-with open(os.path.join(dirname, "github_response.json"), encoding="utf-8") as fp:
-    github_response = json.load(fp)
+with open(os.path.join(dirname, "github_pulls_response.json"), encoding="utf-8") as fp:
+    github_pulls_response = json.load(fp)
+with open(os.path.join(dirname, "github_issues_response.json"), encoding="utf-8") as fp:
+    github_issues_response = json.load(fp)
 with open(os.path.join(dirname, "github_single_response.json"), encoding="utf-8") as fp:
     github_single_response = json.load(fp)