Use metadata to reconcile go-github with GitHub's OpenAPI descriptions (#2919)

Author: WillAbides

.codecov.yml

@@ -3,5 +3,5 @@ ignore:
   - "github/github-accessors.go"
   # ignore experimental scrape package
   - "scrape"
-  # ignore update-urls
-  - "update-urls"
+  # ignore tools
+  - "tools"

.github/dependabot.yml

@@ -9,7 +9,7 @@ updates:
   schedule:
     interval: weekly
 - package-ecosystem: gomod
-  directory: update-urls
+  directory: tools
   schedule:
     interval: weekly
 - package-ecosystem: github-actions

.github/workflows/linter.yml

@@ -14,3 +14,6 @@ jobs:
         go-version: 1.x
         cache-dependency-path: "**/go.sum"
     - run: script/lint.sh
+      env:
+        CHECK_GITHUB_OPENAPI: 1
+        GITHUB_TOKEN: ${{ github.token }}

.golangci.yml

@@ -50,4 +50,4 @@ issues:
     # We don't care about file inclusion via variable in examples or internal tools.
     - linters: [ gosec ]
       text: 'G304: Potential file inclusion via variable'
-      path: '^(example|update-urls)\/'
+      path: '^(example|tools)\/'

CONTRIBUTING.md

@@ -1,10 +1,9 @@
-# How to contribute #
+# How to contribute
 
 We'd love to accept your patches and contributions to this project. There are
 a just a few small guidelines you need to follow.
 
-
-## Contributor License Agreement ##
+## Contributor License Agreement
 
 Contributions to any Google project must be accompanied by a Contributor
 License Agreement. This is not a copyright **assignment**, it simply gives
@@ -17,7 +16,7 @@ You generally only need to submit a CLA once, so if you've already submitted one
 again.
 
 
-## Reporting issues ##
+## Reporting issues
 
 Bugs, feature requests, and development-related questions should be directed to
 our [GitHub issue tracker](https://github.com/google/go-github/issues).  If
@@ -29,7 +28,7 @@ how the requested feature would help you do that.
 Security related bugs can either be reported in the issue tracker, or if they
 are more sensitive, emailed to <[email protected]>.
 
-## Submitting a patch ##
+## Submitting a patch
 
 1. It's generally best to start by opening a new issue describing the bug or
    feature you're intending to fix. Even if you think it's relatively minor,
@@ -73,7 +72,112 @@ are more sensitive, emailed to <[email protected]>.
 [pull request]: https://help.github.com/articles/creating-a-pull-request
 [monitored by codecov.io]: https://codecov.io/gh/google/go-github
 
-## Scripts ##
+## Code Comments
+
+Every exported method needs to have code comments that follow
+[Go Doc Comments][]. A typical method's comments will look like this:
+
+```go
+// Get fetches a repository.
+//
+// GitHub API docs: https://docs.github.com/rest/repos/repos#get-a-repository
+//
+//meta:operation GET /repos/{owner}/{repo}
+func (s *RepositoriesService) Get(ctx context.Context, owner, repo string) (*Repository, *Response, error) {
+u := fmt.Sprintf("repos/%v/%v", owner, repo)
+req, err := s.client.NewRequest("GET", u, nil)
+...
+}
+```
+
+The first line is the name of the method followed by a short description. This
+could also be a longer description if needed, but there is no need to repeat any
+details that are documented in GitHub's documentation because users are expected
+to follow the documentation links to learn more.
+
+After the description comes a link to the GitHub API documentation. This is
+added or fixed automatically when you run `script/generate.sh`, so you won't
+need to set this yourself.
+
+Finally, the `//meta:operation` comment is a directive to the code generator
+that maps the method to the corresponding OpenAPI operation. Once again, there
+can be multiple directives for methods that call multiple
+endpoints. `script/generate.sh` will normalize these directives for you, so if
+you are adding a new method you can use the pattern from the `u := fmt.Sprintf`
+line instead of looking up what the url parameters are called in the OpenAPI
+description.
+
+[Go Doc Comments]: https://go.dev/doc/comment
+
+## Metadata
+
+GitHub publishes [OpenAPI descriptions of their API][]. We use these
+descriptions to keep documentation links up to date and to keep track of which
+methods call which endpoints via the `//meta:operation` comments described
+above. GitHub's descriptions are far too large to keep in this repository or to
+pull down every time we generate code, so we keep only the metadata we need
+in `openapi_operations.yaml`.
+
+### openapi_operations.yaml
+
+Most contributors won't need to interact with `openapi_operations.yaml`, but it
+may be useful to know what it is. Its sections are:
+
+- `openapi_operations` - is the metadata that comes from GitHub's OpenAPI
+  descriptions. It is generated by `script/metadata.sh update-openapi` and
+  should not be edited by hand. In the rare case where it needs to be
+  overridden, use the `operation_overrides` section instead.
+
+  An operation consists of `name`, `documentation_url`,
+  and `openapi_files`. `openapi_files` is the list of files where the operation
+  is described. In order or preference, values can be "api.github.com.json" for
+  operations available on the free plan, "ghec.json" for operations available on
+  GitHub Enterprise Cloud or "ghes-<version>.json" for operations available on
+  GitHub Enterprise Server. When an operation is described in multiple ghes
+  files, only the most recent version is included. `documentation_url` is the
+  URL that should be linked from godoc. It is the documentation link found in
+  the first file listed in `openapi_files`.
+
+- `openapi_commit` - is the git commit that `script/metadata.sh update-openapi`
+  saw when it last updated `openapi_operations`. It is not necessarily the most
+  recent commit seen because `update-openapi` doesn't update the file when
+  there are no changes to `openapi_operations`.
+
+- `operations` - contains manually added metadata that is not in GitHub's
+  OpenAPI descriptions. There are only a few of these. Some have
+  documentation_urls that point to relevant GitHub documentation that is not in
+  the OpenAPI descriptions. Others have no documentation_url and result in a
+  note in the generated code that the documentation is missing.
+
+- `operation_overrides` - is where we override the documentation_url for 
+  operations where the link in the OpenAPI descriptions is wrong.
+
+### tools/metadata
+
+The `tools/metadata` package is a command-line tool for working with metadata.
+In a typical workflow, you won't use it directly, but you will use it indirectly
+through `script/generate.sh` and `script/lint.sh`.
+
+Its subcommands are:
+
+- `update-openapi` - updates `openapi_operations.yaml` with the latest
+  information from GitHub's OpenAPI descriptions. With `--validate` it will
+  validate that the descriptions are correct as of the commit
+  in `openapi_commit`. `update-openapi --validate` is called
+  by `script/lint.sh`.
+
+- `update-go` - updates Go files with documentation URLs and formats comments.
+  It is used by `script/generate.sh`.
+
+- `format` - formats whitespace in `openapi_operations.yaml` and sorts its
+  arrays. It is used by `script/fmt.sh`.
+
+- `unused` - lists operations from `openapi_operations.yaml` that are not mapped
+  from any methods.
+
+[OpenAPI descriptions of their API]: https://github.com/github/rest-api-description
+
+## Scripts
 
 The `script` directory has shell scripts that help with common development
 tasks.
@@ -86,6 +190,9 @@ tasks.
 **script/lint.sh** runs linters on the project and checks generated files are
 current.
 
+**script/metadata.sh** runs `tools/metadata`. See the [Metadata](#metadata)
+section for more information.
+
 **script/test.sh** runs tests on all modules.
 
 ## Other notes on code organization ##
@@ -104,7 +211,7 @@ defined at <https://docs.github.com/en/rest/webhooks/repos> live in
 [repos_hooks.go]: https://github.com/google/go-github/blob/master/github/repos_hooks.go
 
 
-## Maintainer's Guide ##
+## Maintainer's Guide
 
 (These notes are mostly only for people merging in pull requests.)
 

github/actions.go

@@ -8,5 +8,5 @@ package github
 // ActionsService handles communication with the actions related
 // methods of the GitHub API.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/
+// GitHub API docs: https://docs.github.com/rest/actions/
 type ActionsService service

github/actions_artifacts.go

@@ -14,7 +14,7 @@ import (
 
 // ArtifactWorkflowRun represents a GitHub artifact's workflow run.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts
 type ArtifactWorkflowRun struct {
 	ID               *int64  `json:"id,omitempty"`
 	RepositoryID     *int64  `json:"repository_id,omitempty"`
@@ -27,7 +27,7 @@ type ArtifactWorkflowRun struct {
 // data between jobs in a workflow and provide storage for data
 // once a workflow is complete.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts
 type Artifact struct {
 	ID                 *int64               `json:"id,omitempty"`
 	NodeID             *string              `json:"node_id,omitempty"`
@@ -44,15 +44,17 @@ type Artifact struct {
 
 // ArtifactList represents a list of GitHub artifacts.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#artifacts
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#artifacts
 type ArtifactList struct {
 	TotalCount *int64      `json:"total_count,omitempty"`
 	Artifacts  []*Artifact `json:"artifacts,omitempty"`
 }
 
 // ListArtifacts lists all artifacts that belong to a repository.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#list-artifacts-for-a-repository
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#list-artifacts-for-a-repository
+//
+//meta:operation GET /repos/{owner}/{repo}/actions/artifacts
 func (s *ActionsService) ListArtifacts(ctx context.Context, owner, repo string, opts *ListOptions) (*ArtifactList, *Response, error) {
 	u := fmt.Sprintf("repos/%v/%v/actions/artifacts", owner, repo)
 	u, err := addOptions(u, opts)
@@ -76,7 +78,9 @@ func (s *ActionsService) ListArtifacts(ctx context.Context, owner, repo string,
 
 // ListWorkflowRunArtifacts lists all artifacts that belong to a workflow run.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#list-workflow-run-artifacts
+//
+//meta:operation GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts
 func (s *ActionsService) ListWorkflowRunArtifacts(ctx context.Context, owner, repo string, runID int64, opts *ListOptions) (*ArtifactList, *Response, error) {
 	u := fmt.Sprintf("repos/%v/%v/actions/runs/%v/artifacts", owner, repo, runID)
 	u, err := addOptions(u, opts)
@@ -100,7 +104,9 @@ func (s *ActionsService) ListWorkflowRunArtifacts(ctx context.Context, owner, re
 
 // GetArtifact gets a specific artifact for a workflow run.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#get-an-artifact
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#get-an-artifact
+//
+//meta:operation GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}
 func (s *ActionsService) GetArtifact(ctx context.Context, owner, repo string, artifactID int64) (*Artifact, *Response, error) {
 	u := fmt.Sprintf("repos/%v/%v/actions/artifacts/%v", owner, repo, artifactID)
 
@@ -120,7 +126,9 @@ func (s *ActionsService) GetArtifact(ctx context.Context, owner, repo string, ar
 
 // DownloadArtifact gets a redirect URL to download an archive for a repository.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#download-an-artifact
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#download-an-artifact
+//
+//meta:operation GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}
 func (s *ActionsService) DownloadArtifact(ctx context.Context, owner, repo string, artifactID int64, maxRedirects int) (*url.URL, *Response, error) {
 	u := fmt.Sprintf("repos/%v/%v/actions/artifacts/%v/zip", owner, repo, artifactID)
 
@@ -144,7 +152,9 @@ func (s *ActionsService) DownloadArtifact(ctx context.Context, owner, repo strin
 
 // DeleteArtifact deletes a workflow run artifact.
 //
-// GitHub API docs: https://docs.github.com/en/rest/actions/artifacts#delete-an-artifact
+// GitHub API docs: https://docs.github.com/rest/actions/artifacts#delete-an-artifact
+//
+//meta:operation DELETE /repos/{owner}/{repo}/actions/artifacts/{artifact_id}
 func (s *ActionsService) DeleteArtifact(ctx context.Context, owner, repo string, artifactID int64) (*Response, error) {
 	u := fmt.Sprintf("repos/%v/%v/actions/artifacts/%v", owner, repo, artifactID)