diff --git a/modules/migration/options.go b/modules/migration/options.go index cf696d8ce4036..11d9e0b4bf2c7 100644 --- a/modules/migration/options.go +++ b/modules/migration/options.go @@ -39,6 +39,19 @@ type MigrateOptions struct { MigrateToRepoID int64 MirrorInterval string `json:"mirror_interval"` + // mirror-only: enable read-only metadata sync of issues/pull requests + // on each mirror update. + SyncIssues bool + SyncPullRequests bool + + // SkipReactions omits per-issue and per-comment reaction fetches. Reactions + // are an N+1 call storm (one request per issue and per comment) that + // dominates the API budget and, on a single transient failure, aborts the + // whole pass. The mirror metadata reflection sets this so large repositories + // complete in a sane window; reactions are the lowest-value metadata for a + // read-only mirror. + SkipReactions bool + AWSAccessKeyID string AWSSecretAccessKey string `json:",omitempty"` diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 5a6079b6e2db0..902c4f02295ec 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -11,6 +11,14 @@ var Migrations = struct { BlockedDomains string AllowLocalNetworks bool SkipTLSVerify bool + // the batched GraphQL fast path (see services/migrations/github_graphql.go), + // cutting rate-limit pressure on large repositories. Off by default; set + // [migrations] USE_GRAPHQL_FOR_MIRROR = true to enable (e.g. for A/B timing). + // SyncReactionsForMirror opts metadata mirror syncs into fetching issue/PR/comment + // reactions too. Off by default because reactions are an N+1 call storm (the lowest + // -value, heaviest metadata for a read-only mirror); set + // [migrations] SYNC_REACTIONS_FOR_MIRROR = true to include them. + SyncReactionsForMirror bool }{ MaxAttempts: 3, RetryBackoff: 3, @@ -25,4 +33,5 @@ func loadMigrationsFrom(rootCfg ConfigProvider) { Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("") Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) + Migrations.SyncReactionsForMirror = sec.Key("SYNC_REACTIONS_FOR_MIRROR").MustBool(false) } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index eca1b15b026a5..9fec5bd7d6705 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -418,6 +418,14 @@ type MigrateRepoOptions struct { Releases bool `json:"releases"` MirrorInterval string `json:"mirror_interval"` + // only used when mirror is true: keep the mirror's issues/pull requests in + // sync with the remote on each mirror update, shown read-only. + SyncIssues bool `json:"sync_issues"` + SyncPullRequests bool `json:"sync_pull_requests"` + + // opt into the batched GraphQL fast path for GitHub migrations (fewer API + // calls / less rate-limit pressure on large repos). REST is the default. + AWSAccessKeyID string `json:"aws_access_key_id"` AWSSecretAccessKey string `json:"aws_secret_access_key"` } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 0e3e68d1e8b2b..5f55cdeef08cf 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -28,6 +28,7 @@ import ( "gitea.dev/services/context" "gitea.dev/services/convert" "gitea.dev/services/migrations" + mirror_service "gitea.dev/services/mirror" notify_service "gitea.dev/services/notify" repo_service "gitea.dev/services/repository" ) @@ -163,6 +164,8 @@ func Migrate(ctx *context.APIContext) { opts.Comments = false opts.PullRequests = false opts.Releases = false + opts.SyncIssues = form.SyncIssues + opts.SyncPullRequests = form.SyncPullRequests } if gitServiceType == api.CodeCommitService { opts.AWSAccessKeyID = form.AWSAccessKeyID @@ -201,6 +204,10 @@ func Migrate(ctx *context.APIContext) { return nil, err } notify_service.MigrateRepository(ctx, doer, repoOwner, migratedRepo) + if opts.Mirror && (opts.SyncIssues || opts.SyncPullRequests) { + log.Info("migrate: initiating immediate metadata sync for mirror [%d] %s/%s", migratedRepo.ID, repoOwner.Name, opts.RepoName) + mirror_service.AddPullMirrorToQueue(migratedRepo.ID) + } return migratedRepo, nil } diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 5b65369dd68b6..3aef09f167647 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -79,6 +79,9 @@ type MigrateRepoForm struct { Releases bool `json:"releases"` MirrorInterval string `json:"mirror_interval"` + SyncIssues bool `json:"sync_issues"` + SyncPullRequests bool `json:"sync_pull_requests"` + AWSAccessKeyID string `json:"aws_access_key_id"` AWSSecretAccessKey string `json:"aws_secret_access_key"` } @@ -109,6 +112,12 @@ type RepoSettingForm struct { Template bool EnablePrune bool + // Pull-mirror metadata sync toggles. Issues and pull requests are the only + // choices; their comments, reviews, labels and milestones come along as + // part of the read-only reflection. + MirrorSyncIssues bool `form:"mirror_sync_issues"` + MirrorSyncPullRequests bool `form:"mirror_sync_pull_requests"` + // Advanced settings EnableCode bool diff --git a/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26per_page%3D2%26sort%3Dcreated%26state%3Dall b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26per_page%3D2%26sort%3Dcreated%26state%3Dall new file mode 100644 index 0000000000000..73154f442c3cf --- /dev/null +++ b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26per_page%3D2%26sort%3Dcreated%26state%3Dall @@ -0,0 +1,24 @@ +Access-Control-Allow-Origin: * +Access-Control-Expose-Headers: ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset +Cache-Control: private, max-age=60, s-maxage=60 +Content-Security-Policy: default-src 'none' +Content-Type: application/json; charset=utf-8 +Etag: W/"4d1d8ecb86bafe76a686d250018cc158745dd6bc794ef651eeeeecbfc1a947bc" +Link: ; rel="next", ; rel="last" +Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin +Strict-Transport-Security: max-age=31536000; includeSubdomains; preload +Vary: Accept, Authorization, Cookie, X-GitHub-OTP,Accept-Encoding, Accept, X-Requested-With +X-Accepted-Github-Permissions: issues=read +X-Content-Type-Options: nosniff +X-Frame-Options: deny +X-Github-Api-Version-Selected: 2022-11-28 +X-Github-Media-Type: github.v3; param=squirrel-girl-preview +X-Github-Request-Id: C4F6:A93E1:6A3A9E:5B260C:69AF24E7 +X-Ratelimit-Limit: 5000 +X-Ratelimit-Remaining: 4931 +X-Ratelimit-Reset: 1773089427 +X-Ratelimit-Resource: core +X-Ratelimit-Used: 69 +X-Xss-Protection: 0 + +[{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/1","repository_url":"https://api.github.com/repos/go-gitea/test_repo","labels_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/labels{/name}","comments_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/comments","events_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/events","html_url":"https://github.com/go-gitea/test_repo/issues/1","id":520479843,"node_id":"MDU6SXNzdWU1MjA0Nzk4NDM=","number":1,"title":"Please add an animated gif icon to the merge button","user":{"login":"guillep2k","id":18600385,"node_id":"MDQ6VXNlcjE4NjAwMzg1","avatar_url":"https://avatars.githubusercontent.com/u/18600385?v=4","gravatar_id":"","url":"https://api.github.com/users/guillep2k","html_url":"https://github.com/guillep2k","followers_url":"https://api.github.com/users/guillep2k/followers","following_url":"https://api.github.com/users/guillep2k/following{/other_user}","gists_url":"https://api.github.com/users/guillep2k/gists{/gist_id}","starred_url":"https://api.github.com/users/guillep2k/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guillep2k/subscriptions","organizations_url":"https://api.github.com/users/guillep2k/orgs","repos_url":"https://api.github.com/users/guillep2k/repos","events_url":"https://api.github.com/users/guillep2k/events{/privacy}","received_events_url":"https://api.github.com/users/guillep2k/received_events","type":"User","user_view_type":"public","site_admin":false},"labels":[{"id":1667254252,"node_id":"MDU6TGFiZWwxNjY3MjU0MjUy","url":"https://api.github.com/repos/go-gitea/test_repo/labels/bug","name":"bug","color":"d73a4a","default":true,"description":"Something isn't working"},{"id":1667254261,"node_id":"MDU6TGFiZWwxNjY3MjU0MjYx","url":"https://api.github.com/repos/go-gitea/test_repo/labels/good%20first%20issue","name":"good first issue","color":"7057ff","default":true,"description":"Good for newcomers"}],"state":"closed","locked":false,"assignees":[],"milestone":{"url":"https://api.github.com/repos/go-gitea/test_repo/milestones/1","html_url":"https://github.com/go-gitea/test_repo/milestone/1","labels_url":"https://api.github.com/repos/go-gitea/test_repo/milestones/1/labels","id":4839941,"node_id":"MDk6TWlsZXN0b25lNDgzOTk0MQ==","number":1,"title":"1.0.0","description":"Milestone 1.0.0","creator":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"open_issues":1,"closed_issues":1,"state":"closed","created_at":"2019-11-12T19:37:08Z","updated_at":"2019-11-12T21:56:17Z","due_on":"2019-11-11T00:00:00Z","closed_at":"2019-11-12T19:45:49Z"},"comments":0,"created_at":"2019-11-09T17:00:29Z","updated_at":"2019-11-12T20:29:53Z","closed_at":"2019-11-12T20:22:22Z","assignee":null,"author_association":"MEMBER","type":null,"active_lock_reason":null,"sub_issues_summary":{"total":0,"completed":0,"percent_completed":0},"issue_dependencies_summary":{"blocked_by":0,"total_blocked_by":0,"blocking":0,"total_blocking":0},"body":"I just want the merge button to hurt my eyes a little. 😝 ","closed_by":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"reactions":{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/reactions","total_count":1,"+1":1,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/timeline","performed_via_github_app":null,"state_reason":"completed","pinned_comment":null},{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/2","repository_url":"https://api.github.com/repos/go-gitea/test_repo","labels_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/labels{/name}","comments_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/comments","events_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/events","html_url":"https://github.com/go-gitea/test_repo/issues/2","id":521799485,"node_id":"MDU6SXNzdWU1MjE3OTk0ODU=","number":2,"title":"Test issue","user":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"labels":[{"id":1667254257,"node_id":"MDU6TGFiZWwxNjY3MjU0MjU3","url":"https://api.github.com/repos/go-gitea/test_repo/labels/duplicate","name":"duplicate","color":"cfd3d7","default":true,"description":"This issue or pull request already exists"}],"state":"closed","locked":false,"assignees":[],"milestone":{"url":"https://api.github.com/repos/go-gitea/test_repo/milestones/2","html_url":"https://github.com/go-gitea/test_repo/milestone/2","labels_url":"https://api.github.com/repos/go-gitea/test_repo/milestones/2/labels","id":4839942,"node_id":"MDk6TWlsZXN0b25lNDgzOTk0Mg==","number":2,"title":"1.1.0","description":"Milestone 1.1.0","creator":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"open_issues":0,"closed_issues":2,"state":"closed","created_at":"2019-11-12T19:37:25Z","updated_at":"2019-11-12T21:39:27Z","due_on":"2019-11-12T00:00:00Z","closed_at":"2019-11-12T19:45:46Z"},"comments":2,"created_at":"2019-11-12T21:00:06Z","updated_at":"2019-11-12T22:07:14Z","closed_at":"2019-11-12T21:01:31Z","assignee":null,"author_association":"MEMBER","type":null,"active_lock_reason":null,"sub_issues_summary":{"total":0,"completed":0,"percent_completed":0},"issue_dependencies_summary":{"blocked_by":0,"total_blocked_by":0,"blocking":0,"total_blocking":0},"body":"This is test issue 2, do not touch!","closed_by":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"reactions":{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/reactions","total_count":6,"+1":1,"-1":1,"laugh":1,"hooray":1,"confused":1,"heart":1,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/timeline","performed_via_github_app":null,"state_reason":"completed","pinned_comment":null}] \ No newline at end of file diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 6c87c89d53665..fc8a2de2c7058 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -612,10 +612,14 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba head = "unknown repository" if pr.IsForkPullRequest() && pr.State != "closed" { - // OK we want to fetch the current head as a branch from its CloneURL + // Use the branch name as baseline — every early return below gets the real + // name instead of 'unknown repository', even when the git fetch fails (which + // is normal on a mirror that doesn't carry fork branches). + if pr.Head.Ref != "" { + head = pr.Head.Ref + } - // 1. Is there a head clone URL available? - // 2. Is there a head ref available? + // Try to fetch the actual ref from the fork's clone URL. if pr.Head.CloneURL == "" || pr.Head.Ref == "" { return head, nil } diff --git a/services/migrations/github.go b/services/migrations/github.go index 35ed7d7abd2f9..a1407bf4da34f 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -52,7 +52,13 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName) - return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName) + downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName) + if err != nil { + return nil, err + } + downloader.SkipReactions = opts.SkipReactions + downloader.useGraphQL = true + return downloader, nil } // GitServiceType returns the type of git service @@ -75,6 +81,30 @@ type GithubDownloaderV3 struct { maxPerPage int SkipReactions bool SkipReviews bool + // issuesCursor is the Link-header `after` cursor for paginating the issues + // list. The issues endpoint caps classic page-number pagination at ~page 100, + // so a large repo's issues must be walked by cursor instead. Carried on the + // downloader (created fresh per sync); reset at the start of each sweep. + issuesCursor string + + // useGraphQL opts the issues stream into the batched GraphQL fast path + // (see github_graphql.go), which fetches an issue plus its comments and + // reactions in one request instead of many. gqlIssuesCursor is that path's + // pagination cursor and gqlComments caches the comments fetched alongside the + // issues so the framework's separate comment phase serves them from memory. + useGraphQL bool + gqlIssuesCursor string + gqlComments map[int64][]*base.Comment + gqlCommentsFlat []*base.Comment + // gqlPRCursor paginates the GraphQL pull-request sweep; gqlReviews caches the + // reviews (with their inline comments) fetched alongside each PR so the + // framework's separate review phase serves them from memory. PR issue-comments + // join gqlComments so the comment phase serves issue and PR comments together. + gqlPRCursor string + gqlReviews map[int64][]*base.Review + // gqlPointsSpent accumulates GitHub's GraphQL points budget spent this run + // (benchmark instrumentation; GraphQL is billed on points, not requests/hr). + gqlPointsSpent int64 } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -97,7 +127,7 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token ) client := &http.Client{ Transport: &oauth2.Transport{ - Base: NewMigrationHTTPTransport(), + Base: newRetryTransport(NewMigrationHTTPTransport()), Source: oauth2.ReuseTokenSource(nil, ts), }, } @@ -113,7 +143,7 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token return proxy.Proxy()(req) } client := &http.Client{ - Transport: transport, + Transport: newRetryTransport(transport), } if err := downloader.addClient(client, baseURL); err != nil { return nil, err @@ -122,6 +152,11 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token return &downloader, nil } +// SupportSyncing returns true if it supports syncing an already-migrated repository +func (g *GithubDownloaderV3) SupportSyncing() bool { + return true +} + // String implements Stringer func (g *GithubDownloaderV3) String() string { return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) @@ -329,7 +364,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith r.Published = rel.PublishedAt.Time } - httpClient := newMigrationHTTPClient() + httpClient := NewMigrationHTTPClient() for _, asset := range rel.Assets { assetID := asset.GetID() // Don't optimize this, for closure we need a local variable TODO: no need to do so in new Golang @@ -418,17 +453,43 @@ func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { + // A one-time migration walks by creation order; a zero time means all issues. + return g.getIssuesSince(ctx, page, perPage, time.Time{}, "created") +} + +// GetNewIssues returns issues updated after the given time, paginated +func (g *GithubDownloaderV3) GetNewIssues(ctx context.Context, page, perPage int, updatedAfter time.Time) ([]*base.Issue, bool, error) { + // A resumable sync walks by UPDATE order so the max updated_unix already + // stored is an exact resume point: everything before it is done, everything + // at/after it still needs syncing. (Walking by creation order would let the + // updated-based watermark skip older-but-recently-touched issues.) + if g.useGraphQL { + // GraphQL fast path: fetches issues + comments + reactions in one request. + return g.getNewIssuesGraphQL(ctx, page, updatedAfter) + } + return g.getIssuesSince(ctx, page, perPage, updatedAfter, "updated") +} + +// getIssuesSince returns issues updated after the given time sorted ascending by +// sortField ("created" or "updated"); a zero time returns all issues +func (g *GithubDownloaderV3) getIssuesSince(ctx context.Context, page, perPage int, since time.Time, sortField string) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } + // Paginate by the Link-header `after` cursor, not a page number: GitHub caps + // classic page-number pagination on the issues endpoint at ~page 100 (~10k + // items), silently truncating a large repository. The cursor has no such cap. + // page<=1 marks the first request of a sweep, so reset the cursor there. + if page <= 1 { + g.issuesCursor = "" + } opt := &github.IssueListByRepoOptions{ - Sort: "created", - Direction: "asc", - State: "all", - ListOptions: github.ListOptions{ - PerPage: perPage, - Page: page, - }, + Sort: sortField, + Direction: "asc", + State: "all", + Since: since, + ListCursorOptions: github.ListCursorOptions{After: g.issuesCursor}, + ListOptions: github.ListOptions{PerPage: perPage}, } allIssues := make([]*base.Issue, 0, perPage) @@ -437,8 +498,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } - log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) + log.Trace("Request get issues cursor=%q got %d, next=%q", g.issuesCursor, len(issues), resp.After) g.setRate(&resp.Rate) + g.issuesCursor = resp.After for _, issue := range issues { if issue.IsPullRequest() { continue @@ -449,32 +511,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( labels = append(labels, convertGithubLabel(l)) } - // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient(ctx) - res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{ - ListOptions: github.ListOptions{ - Page: i, - PerPage: perPage, - }, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } + reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage) + if err != nil { + return nil, false, err } var assignees []string @@ -502,7 +541,10 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( }) } - return allIssues, len(issues) < perPage, nil + // End when the cursor is exhausted AND a short page confirms no more results. + // Both conditions together handle cursor-based and page-based responses. + isEnd := resp.After == "" && len(issues) < perPage + return allIssues, isEnd, nil } // SupportGetRepoComments return true if it supports get repo comments @@ -517,6 +559,12 @@ func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.C } func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) { + return g.getCommentsSince(ctx, commentable, nil) +} + +// getCommentsSince returns an issue's or pull request's comments; a non-nil +// since returns only those updated at or after it +func (g *GithubDownloaderV3) getCommentsSince(ctx context.Context, commentable base.Commentable, since *time.Time) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -525,6 +573,7 @@ func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.C opt := &github.IssueListCommentsOptions{ Sort: &created, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ PerPage: g.maxPerPage, }, @@ -587,17 +636,24 @@ func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.C // GetAllComments returns repository comments according page and perPageSize func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) { + // A one-time migration walks by creation order. + return g.getAllCommentsSince(ctx, page, perPage, nil, "created") +} + +// getAllCommentsSince returns all repository issue and pull request comments +// paginated; a non-nil since returns only those updated at or after it +func (g *GithubDownloaderV3) getAllCommentsSince(ctx context.Context, page, perPage int, since *time.Time, sortField string) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) - created = "created" asc = "asc" ) if perPage > g.maxPerPage { perPage = g.maxPerPage } opt := &github.IssueListCommentsOptions{ - Sort: &created, + Sort: &sortField, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ Page: page, PerPage: perPage, @@ -682,83 +738,157 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) g.setRate(&resp.Rate) for _, pr := range prs { - labels := make([]*base.Label, 0, len(pr.Labels)) - for _, l := range pr.Labels { - labels = append(labels, convertGithubLabel(l)) + basePR, err := g.convertGithubPullRequest(ctx, pr, perPage) + if err != nil { + return nil, false, err } + allPRs = append(allPRs, basePR) - // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient(ctx) - res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{ - ListOptions: github.ListOptions{ - Page: i, - PerPage: perPage, - }, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } - } + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(basePR, g.baseURL, g) + } - // download patch and saved as tmp file - g.waitAndPickClient(ctx) + // Terminate on the Link header, not len(prs) < perPage (see getIssuesSince): + // a short page mid-results must not be misread as the end of a large backfill. + return allPRs, resp.NextPage == 0, nil +} - allPRs = append(allPRs, &base.PullRequest{ - Title: pr.GetTitle(), - Number: int64(pr.GetNumber()), - PosterID: pr.GetUser().GetID(), - PosterName: pr.GetUser().GetLogin(), - PosterEmail: pr.GetUser().GetEmail(), - Content: pr.GetBody(), - Milestone: pr.GetMilestone().GetTitle(), - State: pr.GetState(), - Created: pr.GetCreatedAt().Time, - Updated: pr.GetUpdatedAt().Time, - Closed: pr.ClosedAt.GetTime(), - Labels: labels, - Merged: pr.MergedAt != nil, - MergeCommitSHA: pr.GetMergeCommitSHA(), - MergedTime: pr.MergedAt.GetTime(), - IsLocked: pr.ActiveLockReason != nil, - Head: base.PullRequestBranch{ - Ref: pr.GetHead().GetRef(), - SHA: pr.GetHead().GetSHA(), - OwnerName: pr.GetHead().GetUser().GetLogin(), - RepoName: pr.GetHead().GetRepo().GetName(), - CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here - }, - Base: base.PullRequestBranch{ - Ref: pr.GetBase().GetRef(), - SHA: pr.GetBase().GetSHA(), - RepoName: pr.GetBase().GetRepo().GetName(), - OwnerName: pr.GetBase().GetUser().GetLogin(), - }, - PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here - Reactions: reactions, - ForeignIndex: int64(*pr.Number), - IsDraft: pr.GetDraft(), - }) +// GetNewPullRequests returns pull requests updated after the given time, paginated. +// The pull request list API has no `since` filter, so it lists by most recently +// updated and stops as soon as a pull request older than updatedAfter appears. +// The search API is deliberately avoided: its results are capped at 1,000 and it +// has a separate, much smaller rate limit. +func (g *GithubDownloaderV3) GetNewPullRequests(ctx context.Context, page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { + if g.useGraphQL { + // GraphQL fast path: fetches PRs + comments + reviews + review comments in + // one batched request instead of the REST per-PR review/comment N+1. + return g.getNewPullRequestsGraphQL(ctx, page, updatedAfter) + } + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } + opt := &github.PullRequestListOptions{ + Sort: "updated", + Direction: "asc", + State: "all", + ListOptions: github.ListOptions{ + PerPage: perPage, + Page: page, + }, + } + allPRs := make([]*base.PullRequest, 0, perPage) + g.waitAndPickClient(ctx) + prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt) + if err != nil { + return nil, false, fmt.Errorf("error while listing pull requests: %w", err) + } + log.Trace("Request get new pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) + g.setRate(&resp.Rate) + for _, pr := range prs { + // Walk ascending by update time and skip what is already synced (older + // than the watermark). The GitHub pull-request list has no server-side + // "since" filter, so the already-done head is skipped client-side; + // paginating to the true end rather than stopping early is what lets a + // partial sweep resume from the max updated_unix already stored. + if pr.GetUpdatedAt().Time.Before(updatedAfter) { + continue + } + basePR, err := g.convertGithubPullRequest(ctx, pr, perPage) + if err != nil { + return nil, false, err + } + allPRs = append(allPRs, basePR) // SECURITY: Ensure that the PR is safe - _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) + _ = CheckAndEnsureSafePR(basePR, g.baseURL, g) } - return allPRs, len(prs) < perPage, nil + // Terminate on the Link header, not len(prs) < perPage (see getIssuesSince): + // a short page mid-results must not be misread as the end of a large backfill. + return allPRs, resp.NextPage == 0, nil +} + +func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) { + labels := make([]*base.Label, 0, len(pr.Labels)) + for _, l := range pr.Labels { + labels = append(labels, convertGithubLabel(l)) + } + + reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage) + if err != nil { + return nil, err + } + + // download patch and saved as tmp file + g.waitAndPickClient(ctx) + + return &base.PullRequest{ + Title: pr.GetTitle(), + Number: int64(pr.GetNumber()), + PosterID: pr.GetUser().GetID(), + PosterName: pr.GetUser().GetLogin(), + PosterEmail: pr.GetUser().GetEmail(), + Content: pr.GetBody(), + Milestone: pr.GetMilestone().GetTitle(), + State: pr.GetState(), + Created: pr.GetCreatedAt().Time, + Updated: pr.GetUpdatedAt().Time, + Closed: pr.ClosedAt.GetTime(), + Labels: labels, + Merged: pr.MergedAt != nil, + MergeCommitSHA: pr.GetMergeCommitSHA(), + MergedTime: pr.MergedAt.GetTime(), + IsLocked: pr.ActiveLockReason != nil, + Head: base.PullRequestBranch{ + Ref: pr.GetHead().GetRef(), + SHA: pr.GetHead().GetSHA(), + OwnerName: pr.GetHead().GetUser().GetLogin(), + RepoName: pr.GetHead().GetRepo().GetName(), + CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here + }, + Base: base.PullRequestBranch{ + Ref: pr.GetBase().GetRef(), + SHA: pr.GetBase().GetSHA(), + RepoName: pr.GetBase().GetRepo().GetName(), + OwnerName: pr.GetBase().GetUser().GetLogin(), + }, + PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here + Reactions: reactions, + ForeignIndex: int64(*pr.Number), + IsDraft: pr.GetDraft(), + }, nil +} + +// getIssueReactions returns the reactions on an issue or pull request +func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) { + var reactions []*base.Reaction + if g.SkipReactions { + return reactions, nil + } + for i := 1; ; i++ { + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, + }) + if err != nil { + return nil, err + } + g.setRate(&resp.Rate) + if len(res) == 0 { + break + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } + return reactions, nil } func convertGithubReview(r *github.PullRequestReview) *base.Review { @@ -822,6 +952,49 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs } // GetReviews returns pull requests review +// nilIfZero returns a pointer to t, or nil when t is the zero time. The comment +// APIs take a *time.Time `since`; a pointer to the zero time would be serialized +// as since=0001-01-01, which GitHub rejects with 422, so a zero time (first +// sync, no watermark) must be sent as nil to omit the filter and fetch all. +func nilIfZero(t time.Time) *time.Time { + if t.IsZero() { + return nil + } + return &t +} + +// GetNewComments returns an issue's or pull request's comments updated at or +// after the given time +func (g *GithubDownloaderV3) GetNewComments(ctx context.Context, commentable base.Commentable, updatedAfter time.Time) ([]*base.Comment, bool, error) { + comments, err := g.getCommentsSince(ctx, commentable, nilIfZero(updatedAfter)) + return comments, false, err +} + +// GetAllNewComments returns all repository comments updated at or after the +// given time, paginated +func (g *GithubDownloaderV3) GetAllNewComments(ctx context.Context, page, perPage int, updatedAfter time.Time) ([]*base.Comment, bool, error) { + if g.useGraphQL { + // GraphQL fast path: comments already came back with their issues; serve + // them from the cache instead of a second round of API calls. + return g.getCachedComments(page, perPage) + } + // A resumable sync walks by UPDATE order so the max comment updated_unix + // already stored is an exact resume point. + return g.getAllCommentsSince(ctx, page, perPage, nilIfZero(updatedAfter), "updated") +} + +// GetNewReviews returns a pull request's reviews updated at or after the given +// time. GitHub's reviews API has no since filter, so all reviews are refetched. +func (g *GithubDownloaderV3) GetNewReviews(ctx context.Context, reviewable base.Reviewable, updatedAfter time.Time) ([]*base.Review, error) { + if g.useGraphQL { + // GraphQL fast path: reviews (and their inline comments) already came back + // with their pull request; serve them from the cache instead of the REST + // per-PR ListReviews + per-review ListReviewComments N+1. + return g.gqlReviews[reviewable.GetForeignIndex()], nil + } + return g.GetReviews(ctx, reviewable) +} + func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) if g.SkipReviews { diff --git a/services/migrations/github_graphql.go b/services/migrations/github_graphql.go new file mode 100644 index 0000000000000..6e01c79c4f2d1 --- /dev/null +++ b/services/migrations/github_graphql.go @@ -0,0 +1,519 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +// Optional GraphQL-backed fast path for the GitHub migration downloader. +// +// The REST downloader makes many small calls — worst of all a separate +// reactions request per issue and per comment — so a large repository exhausts +// the 5,000 req/hr limit long before it finishes. GitHub's GraphQL API returns +// an issue together with its comments, reactions and labels in a single request +// (100 nodes/page) and is billed by a separate points budget whose cost model is +// far more forgiving for this batched shape. This collapses dozens of REST calls +// into one. +// +// GraphQL is the default path for GitHub migrations. The REST downloader is +// kept as fallback. Scope: the issues stream (issues + their comments + reactions + +// labels). Pull-request reviews/review-threads over GraphQL are a TODO — they +// still go through the REST path. Comment/reaction pages beyond the first 100 for +// a single entity fall back to REST for that entity so nothing is silently +// dropped. + +import ( + "bytes" + "context" + "fmt" + "net/http" + "slices" + "strconv" + "strings" + "time" + + "gitea.dev/modules/json" + "gitea.dev/modules/log" + base "gitea.dev/modules/migration" +) + +const ( + // GraphQL returns at most 100 nodes per connection page. + githubGraphQLPageSize = 100 + // Pause when the GraphQL points budget drops to this, mirroring the REST + // GithubLimitRateRemaining guard. + githubGraphQLPointsFloor = 10 + // maxGraphQLRateRetries bounds how many times a single request waits out a + // RATE_LIMITED response before giving up, so a persistently throttled sync + // fails loudly instead of looping forever. + maxGraphQLRateRetries = 5 + // graphQLRateResetCap bounds how long a single RATE_LIMITED wait can sleep, + // guarding against a bogus reset header. + graphQLRateResetCap = time.Hour + // graphQLRateResetFallback is used when a RATE_LIMITED response carries no + // usable reset header. + graphQLRateResetFallback = time.Minute +) + +// graphQLRequest is the POST body for a GraphQL call. +type graphQLRequest struct { + Query string `json:"query"` + Variables map[string]any `json:"variables"` +} + +// graphQLError is one entry in a GraphQL response's top-level errors array. +type graphQLError struct { + Type string `json:"type"` + Message string `json:"message"` +} + +func (e graphQLError) Error() string { + if e.Type != "" { + return e.Type + ": " + e.Message + } + return e.Message +} + +// graphQLRateLimit is GitHub's `rateLimit { ... }` block (the points budget, +// distinct from the REST 5,000 req/hr limit). +type graphQLRateLimit struct { + Cost int `json:"cost"` + Remaining int `json:"remaining"` + ResetAt time.Time `json:"resetAt"` +} + +// graphQLEndpoint derives the GraphQL URL from the REST base URL. github.com uses +// a dedicated host; GitHub Enterprise serves it at {base}/api/graphql. +func (g *GithubDownloaderV3) graphQLEndpoint() string { + if g.baseURL == "" || g.baseURL == "https://github.com" { + return "https://api.github.com/graphql" + } + return strings.TrimSuffix(g.baseURL, "/") + "/api/graphql" +} + +// doGraphQL executes a query and unmarshals the `data` field into out. It reuses +// the currently selected client's authenticated HTTP client (so it inherits the +// oauth2 token and the retrying transport), and returns the response's rateLimit +// block for budget accounting. +func (g *GithubDownloaderV3) doGraphQL(ctx context.Context, query string, vars map[string]any, out any) error { + payload, err := json.Marshal(graphQLRequest{Query: query, Variables: vars}) + if err != nil { + return err + } + + for attempt := 0; ; attempt++ { + g.waitAndPickClient(ctx) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.graphQLEndpoint(), bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := g.getClient().Client().Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return fmt.Errorf("github graphql: unexpected status %s", resp.Status) + } + + var envelope struct { + Data json.Value `json:"data"` + Errors []graphQLError `json:"errors"` + } + decodeErr := json.NewDecoder(resp.Body).Decode(&envelope) + header := resp.Header + resp.Body.Close() + if decodeErr != nil { + return decodeErr + } + + if len(envelope.Errors) > 0 { + // A RATE_LIMITED response means the points budget (or a secondary + // abuse limit) is exhausted. Wait out the window and retry in place + // rather than aborting the whole metadata sync — the REST path does + // the same via waitAndPickClient. + if isGraphQLRateLimited(envelope.Errors) && attempt < maxGraphQLRateRetries { + if g.waitForGraphQLRateReset(ctx, header) { + continue + } + } + return fmt.Errorf("github graphql: %w", envelope.Errors[0]) + } + return json.Unmarshal(envelope.Data, out) + } +} + +// isGraphQLRateLimited reports whether any of the returned errors is GitHub's +// RATE_LIMITED type (points budget or secondary abuse limit exhausted). +func isGraphQLRateLimited(errs []graphQLError) bool { + for _, e := range errs { + if e.Type == "RATE_LIMITED" { + return true + } + } + return false +} + +// waitForGraphQLRateReset sleeps until the GitHub rate-limit window resets so a +// RATE_LIMITED response is waited out in-process. Returns false if the context is +// cancelled while waiting. +func (g *GithubDownloaderV3) waitForGraphQLRateReset(ctx context.Context, header http.Header) bool { + wait := graphQLRateResetWait(header) + log.Info("github graphql: RATE_LIMITED, sleeping %s until reset", wait.Round(time.Second)) + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +// graphQLRateResetWait derives how long to wait from GitHub's rate headers: +// Retry-After (secondary limits, in seconds) takes precedence, then +// X-RateLimit-Reset (primary budget, a unix epoch). It falls back to a short +// delay when neither is present and is capped to guard against a bogus header. +func graphQLRateResetWait(header http.Header) time.Duration { + wait := graphQLRateResetFallback + if ra := header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + wait = time.Duration(secs) * time.Second + } + } else if reset := header.Get("X-RateLimit-Reset"); reset != "" { + if epoch, err := strconv.ParseInt(reset, 10, 64); err == nil { + if d := time.Until(time.Unix(epoch, 0)); d > 0 { + wait = d + } + } + } + if wait > graphQLRateResetCap { + wait = graphQLRateResetCap + } + return wait +} + +// respectGraphQLBudget pauses until the points budget resets when it is nearly +// exhausted, so a long backfill sleeps in-process rather than erroring. +func (g *GithubDownloaderV3) respectGraphQLBudget(ctx context.Context, rl graphQLRateLimit) { + // Benchmark instrumentation: GraphQL is billed on a points budget (not the REST + // requests/hr limit), so record per-query cost and the running total for the run. + g.gqlPointsSpent += int64(rl.Cost) + log.Trace("github graphql: query cost=%d remaining=%d cumulative=%d", rl.Cost, rl.Remaining, g.gqlPointsSpent) + + if rl.Remaining > githubGraphQLPointsFloor || rl.ResetAt.IsZero() { + return + } + wait := time.Until(rl.ResetAt) + if wait <= 0 { + return + } + log.Info("github graphql: points budget low (%d), sleeping %s until reset", rl.Remaining, wait.Round(time.Second)) + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +// --- issues query ----------------------------------------------------------- + +// graphQLIssuesQuery builds the issues query. GitHub costs a query by the product +// of first: down each path and rejects anything over 500,000 possible nodes, so +// nesting reactions under comments — issues(100)×comments(100)×reactions(100) = +// 1,000,000 — is refused outright (MAX_NODE_LIMIT_EXCEEDED). Reactions are also +// exactly what a mirror skips (SkipReactions), so we never nest them under +// comments and include the cheap issue-level reactions only when reactions are +// actually wanted (a non-mirror migration). This keeps the query at ~16k nodes +// (skip) / ~26k (with reactions), far under the ceiling. +func (g *GithubDownloaderV3) graphQLIssuesQuery() string { + issueReactions := "" + if !g.SkipReactions { + issueReactions = "reactions(first:100){nodes{content user{login ... on User{databaseId}}}}" + } + return fmt.Sprintf(` +query($owner:String!,$name:String!,$cursor:String,$since:DateTime){ + repository(owner:$owner,name:$name){ + issues(first:100,after:$cursor,orderBy:{field:UPDATED_AT,direction:ASC},filterBy:{since:$since}){ + pageInfo{hasNextPage endCursor} + nodes{ + id number title body state createdAt updatedAt closedAt + locked + author{login ... on User{databaseId}} + milestone{title} + labels(first:30){nodes{name color description}} + assignees(first:30){nodes{login}} + %s + comments(first:100){ + totalCount + nodes{ + databaseId body createdAt updatedAt + author{login ... on User{databaseId}} + } + } + } + } + } + rateLimit{cost remaining resetAt} +}`, issueReactions) +} + +// gqlActor is a GitHub Actor; databaseId is only populated for real users. +type gqlActor struct { + Login string `json:"login"` + DatabaseID int64 `json:"databaseId"` +} + +type gqlReactionConn struct { + Nodes []struct { + Content string `json:"content"` + User gqlActor `json:"user"` + } `json:"nodes"` +} + +type gqlComment struct { + DatabaseID int64 `json:"databaseId"` + Body string `json:"body"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + Author gqlActor `json:"author"` + Reactions gqlReactionConn `json:"reactions"` +} + +type gqlIssue struct { + ID string `json:"id"` // GraphQL node id, for the timeline node-id pass + Number int64 `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + State string `json:"state"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ClosedAt *time.Time `json:"closedAt"` + Locked bool `json:"locked"` // the real locked boolean (activeLockReason is null when locked without a reason) + Author gqlActor `json:"author"` + Milestone *struct { + Title string `json:"title"` + } `json:"milestone"` + Labels struct { + Nodes []struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + } `json:"nodes"` + } `json:"labels"` + Assignees struct { + Nodes []struct { + Login string `json:"login"` + } `json:"nodes"` + } `json:"assignees"` + Reactions gqlReactionConn `json:"reactions"` + Comments struct { + TotalCount int `json:"totalCount"` + Nodes []gqlComment `json:"nodes"` + } `json:"comments"` +} + +type gqlIssuesResponse struct { + Repository struct { + Issues struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + Nodes []gqlIssue `json:"nodes"` + } `json:"issues"` + } `json:"repository"` + RateLimit graphQLRateLimit `json:"rateLimit"` +} + +// getNewIssuesGraphQL fetches a page of issues (updated-ascending, resumable like +// the REST path) together with their comments and reactions in a single request, +// caching the comments so the framework's separate comment phase serves them from +// memory instead of re-fetching. Returns the issues, whether this was the last +// page, and an error. +// +// page<=1 marks the first request of a sweep and resets the cursor, matching +// getIssuesSince. +func (g *GithubDownloaderV3) getNewIssuesGraphQL(ctx context.Context, page int, since time.Time) ([]*base.Issue, bool, error) { + if page <= 1 { + g.gqlIssuesCursor = "" + g.gqlComments = map[int64][]*base.Comment{} + if since.IsZero() { + log.Info("metadata sync [%s/%s]: issues — full sweep (no watermark)", g.repoOwner, g.repoName) + } else { + log.Info("metadata sync [%s/%s]: issues — incremental since %s", g.repoOwner, g.repoName, since.Format(time.RFC3339)) + } + } + + vars := map[string]any{ + "owner": g.repoOwner, + "name": g.repoName, + } + if g.gqlIssuesCursor != "" { + vars["cursor"] = g.gqlIssuesCursor + } + if !since.IsZero() { + vars["since"] = since.Format(time.RFC3339) + } + + var resp gqlIssuesResponse + if err := g.doGraphQL(ctx, g.graphQLIssuesQuery(), vars, &resp); err != nil { + return nil, false, err + } + g.respectGraphQLBudget(ctx, resp.RateLimit) + g.gqlIssuesCursor = resp.Repository.Issues.PageInfo.EndCursor + + issues := make([]*base.Issue, 0, len(resp.Repository.Issues.Nodes)) + // issue node id -> number, for the timeline node-id pass after this page + timelineTargets := map[string]int64{} + for i := range resp.Repository.Issues.Nodes { + node := &resp.Repository.Issues.Nodes[i] + + labels := make([]*base.Label, 0, len(node.Labels.Nodes)) + for _, l := range node.Labels.Nodes { + labels = append(labels, &base.Label{Name: l.Name, Color: l.Color, Description: l.Description}) + } + assignees := make([]string, 0, len(node.Assignees.Nodes)) + for _, a := range node.Assignees.Nodes { + assignees = append(assignees, a.Login) + } + var milestone string + if node.Milestone != nil { + milestone = node.Milestone.Title + } + + issues = append(issues, &base.Issue{ + Number: node.Number, + Title: node.Title, + Content: node.Body, + PosterID: node.Author.DatabaseID, + PosterName: node.Author.Login, + State: strings.ToLower(node.State), + Milestone: milestone, + Created: node.CreatedAt, + Updated: node.UpdatedAt, + Closed: node.ClosedAt, + IsLocked: node.Locked, + Labels: labels, + Assignees: assignees, + Reactions: convertGraphQLReactions(node.Reactions), + ForeignIndex: node.Number, + }) + timelineTargets[node.ID] = node.Number + + // Cache the comments that came back with the issue so the comment phase + // serves them for free. If the issue has more than one page of comments, + // fall back to REST for the complete set (correctness over the fast path). + if node.Comments.TotalCount > len(node.Comments.Nodes) { + rest, err := g.getComments(ctx, &base.Issue{Number: node.Number, ForeignIndex: node.Number}) + if err != nil { + return nil, false, err + } + g.gqlComments[node.Number] = rest + continue + } + g.gqlComments[node.Number] = convertGraphQLComments(node.Number, node.Comments.Nodes) + } + + log.Info("metadata sync [%s/%s]: issues page %d — fetched %d issues, %d with timeline targets", + g.repoOwner, g.repoName, page, len(issues), len(timelineTargets)) + + // Timeline events are a best-effort enrichment fetched in a separate sweep; + // a failure here must never abort the issue/PR/comment import (see #37). + if len(timelineTargets) > 0 { + if err := g.attachTimelineEvents(ctx, timelineTargets); err != nil { + log.Error("github graphql: timeline events sync failed, importing without them: %v", err) + } + } + + return issues, !resp.Repository.Issues.PageInfo.HasNextPage, nil +} + +func convertGraphQLComments(issueNumber int64, nodes []gqlComment) []*base.Comment { + comments := make([]*base.Comment, 0, len(nodes)) + for _, c := range nodes { + comments = append(comments, &base.Comment{ + IssueIndex: issueNumber, + Index: c.DatabaseID, + PosterID: c.Author.DatabaseID, + PosterName: c.Author.Login, + Content: c.Body, + Created: c.CreatedAt, + Updated: c.UpdatedAt, + Reactions: convertGraphQLReactions(c.Reactions), + }) + } + return comments +} + +// convertGraphQLReactions maps GitHub's GraphQL reaction content enum +// (THUMBS_UP, …) onto the REST/Gitea content strings (+1, …). +func convertGraphQLReactions(conn gqlReactionConn) []*base.Reaction { + if len(conn.Nodes) == 0 { + return nil + } + reactions := make([]*base.Reaction, 0, len(conn.Nodes)) + for _, r := range conn.Nodes { + reactions = append(reactions, &base.Reaction{ + UserID: r.User.DatabaseID, + UserName: r.User.Login, + Content: graphQLReactionContent(r.Content), + }) + } + return reactions +} + +// getCachedComments serves the comments gathered by the GraphQL issue sweep, +// paginated to match the framework's comment phase. It flattens the per-issue +// cache once (deterministically ordered by issue number) and returns slices of +// it, so the comment phase makes no additional API calls. +func (g *GithubDownloaderV3) getCachedComments(page, perPage int) ([]*base.Comment, bool, error) { + if g.gqlCommentsFlat == nil { + nums := make([]int64, 0, len(g.gqlComments)) + for n := range g.gqlComments { + nums = append(nums, n) + } + slices.Sort(nums) + // non-nil sentinel so an empty sweep isn't re-flattened every call + g.gqlCommentsFlat = make([]*base.Comment, 0) + for _, n := range nums { + g.gqlCommentsFlat = append(g.gqlCommentsFlat, g.gqlComments[n]...) + } + } + if perPage <= 0 { + perPage = githubGraphQLPageSize + } + start := (page - 1) * perPage + if start < 0 || start >= len(g.gqlCommentsFlat) { + return nil, true, nil + } + end := min(start+perPage, len(g.gqlCommentsFlat)) + return g.gqlCommentsFlat[start:end], end >= len(g.gqlCommentsFlat), nil +} + +func graphQLReactionContent(enum string) string { + switch enum { + case "THUMBS_UP": + return "+1" + case "THUMBS_DOWN": + return "-1" + case "LAUGH": + return "laugh" + case "HOORAY": + return "hooray" + case "CONFUSED": + return "confused" + case "HEART": + return "heart" + case "ROCKET": + return "rocket" + case "EYES": + return "eyes" + default: + return strings.ToLower(enum) + } +} diff --git a/services/migrations/github_graphql_pr.go b/services/migrations/github_graphql_pr.go new file mode 100644 index 0000000000000..7bf0a231c2d68 --- /dev/null +++ b/services/migrations/github_graphql_pr.go @@ -0,0 +1,409 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +// GraphQL fast path for pull requests — the follow-on to the issues path in +// github_graphql.go. It fetches a pull request together with its comments, +// reviews and review (inline) comments in a single batched request instead of +// the REST path's per-PR ListReviews + per-review ListReviewComments N+1. +// +// Reviews are cached so the framework's separate review phase serves them from +// memory; a PR's issue-comments join the shared comment cache so the comment +// phase serves issue and PR comments together. The patch URL is constructed (a +// raw download, not an API call), so no API budget is spent resolving it. +// Entities that exceed one page (issue comments, reviews, or a review's inline +// comments) fall back to REST for that pull request so nothing is dropped. + +import ( + "context" + "fmt" + "time" + + "gitea.dev/modules/log" + base "gitea.dev/modules/migration" +) + +// graphQLPullRequestsQuery builds the PR query. Same 500k-node ceiling as the +// issues query: nesting reactions under comments (pullRequests×comments×reactions) +// is both the biggest node-cost path and exactly the metadata a mirror skips, so +// reactions are never nested under comments; the cheap PR-level reactions are +// included only when reactions are wanted. This keeps the query ~55k nodes (the +// dominant path is reviews(50)×comments(50) = 50k), far under the ceiling. +func (g *GithubDownloaderV3) graphQLPullRequestsQuery() string { + prReactions := "" + if !g.SkipReactions { + prReactions = "reactions(first:100){nodes{content user{login ... on User{databaseId}}}}" + } + return fmt.Sprintf(` +query($owner:String!,$name:String!,$cursor:String){ + repository(owner:$owner,name:$name){ + pullRequests(first:20,after:$cursor,orderBy:{field:UPDATED_AT,direction:DESC},states:[OPEN,CLOSED,MERGED]){ + pageInfo{hasNextPage endCursor} + nodes{ + id number title body state createdAt updatedAt closedAt mergedAt isDraft + locked + author{login ... on User{databaseId}} + milestone{title} + mergeCommit{oid} + headRefName headRefOid baseRefName baseRefOid + headRepository{name url owner{login}} + baseRepository{name owner{login}} + labels(first:30){nodes{name color description}} + assignees(first:30){nodes{login}} + %s + comments(first:100){totalCount nodes{databaseId body createdAt updatedAt author{login ... on User{databaseId}}}} + reviews(first:50){totalCount nodes{ + databaseId state body createdAt submittedAt + author{login ... on User{databaseId}} + commit{oid} + comments(first:50){totalCount nodes{databaseId body path diffHunk position commit{oid} author{login ... on User{databaseId}} createdAt updatedAt replyTo{databaseId}}} + }} + reviewRequests(first:50){nodes{requestedReviewer{... on User{login databaseId}}}} + } + } + } + rateLimit{cost remaining resetAt} +}`, prReactions) +} + +type gqlRepoRef struct { + Name string `json:"name"` + URL string `json:"url"` + Owner gqlActor `json:"owner"` +} + +type gqlReviewComment struct { + DatabaseID int64 `json:"databaseId"` + Body string `json:"body"` + Path string `json:"path"` + DiffHunk string `json:"diffHunk"` + Position int `json:"position"` + Commit *struct { + OID string `json:"oid"` + } `json:"commit"` + Author gqlActor `json:"author"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ReplyTo *struct { + DatabaseID int64 `json:"databaseId"` + } `json:"replyTo"` +} + +type gqlReview struct { + DatabaseID int64 `json:"databaseId"` + State string `json:"state"` + Body string `json:"body"` + CreatedAt time.Time `json:"createdAt"` + SubmittedAt *time.Time `json:"submittedAt"` + Author gqlActor `json:"author"` + Commit *struct { + OID string `json:"oid"` + } `json:"commit"` + Comments struct { + TotalCount int `json:"totalCount"` + Nodes []gqlReviewComment `json:"nodes"` + } `json:"comments"` +} + +type gqlPullRequest struct { + ID string `json:"id"` // GraphQL node id, for the timeline node-id pass + Number int64 `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + State string `json:"state"` // OPEN, CLOSED, MERGED + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ClosedAt *time.Time `json:"closedAt"` + MergedAt *time.Time `json:"mergedAt"` + IsDraft bool `json:"isDraft"` + Locked bool `json:"locked"` // the real locked boolean (activeLockReason is null when locked without a reason) + Author gqlActor `json:"author"` + Milestone *struct { + Title string `json:"title"` + } `json:"milestone"` + MergeCommit *struct { + OID string `json:"oid"` + } `json:"mergeCommit"` + HeadRefName string `json:"headRefName"` + HeadRefOID string `json:"headRefOid"` + BaseRefName string `json:"baseRefName"` + BaseRefOID string `json:"baseRefOid"` + HeadRepo *gqlRepoRef `json:"headRepository"` + BaseRepo *gqlRepoRef `json:"baseRepository"` + Labels struct { + Nodes []struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + } `json:"nodes"` + } `json:"labels"` + Assignees struct { + Nodes []struct { + Login string `json:"login"` + } `json:"nodes"` + } `json:"assignees"` + Reactions gqlReactionConn `json:"reactions"` + Comments struct { + TotalCount int `json:"totalCount"` + Nodes []gqlComment `json:"nodes"` + } `json:"comments"` + Reviews struct { + TotalCount int `json:"totalCount"` + Nodes []gqlReview `json:"nodes"` + } `json:"reviews"` + ReviewRequests struct { + Nodes []struct { + RequestedReviewer gqlActor `json:"requestedReviewer"` + } `json:"nodes"` + } `json:"reviewRequests"` +} + +type gqlPullRequestsResponse struct { + Repository struct { + PullRequests struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + Nodes []gqlPullRequest `json:"nodes"` + } `json:"pullRequests"` + } `json:"repository"` + RateLimit graphQLRateLimit `json:"rateLimit"` +} + +// getNewPullRequestsGraphQL fetches a page of pull requests (updated-ascending, +// resumable like the REST path) together with their comments, reviews and review +// comments in one request. Comments are cached alongside the issue comments and +// reviews are cached per PR for the framework's later phases. page<=1 marks the +// first request of a sweep. +func (g *GithubDownloaderV3) getNewPullRequestsGraphQL(ctx context.Context, page int, since time.Time) ([]*base.PullRequest, bool, error) { + if page <= 1 { + g.gqlPRCursor = "" + g.gqlReviews = map[int64][]*base.Review{} + if since.IsZero() { + log.Info("metadata sync [%s/%s]: pull requests — full sweep (no watermark)", g.repoOwner, g.repoName) + } else { + log.Info("metadata sync [%s/%s]: pull requests — newest-first, stopping at watermark %s", g.repoOwner, g.repoName, since.Format(time.RFC3339)) + } + } + if g.gqlComments == nil { + g.gqlComments = map[int64][]*base.Comment{} + } + + vars := map[string]any{"owner": g.repoOwner, "name": g.repoName} + if g.gqlPRCursor != "" { + vars["cursor"] = g.gqlPRCursor + } + + var resp gqlPullRequestsResponse + if err := g.doGraphQL(ctx, g.graphQLPullRequestsQuery(), vars, &resp); err != nil { + return nil, false, err + } + g.respectGraphQLBudget(ctx, resp.RateLimit) + g.gqlPRCursor = resp.Repository.PullRequests.PageInfo.EndCursor + + allPRs := make([]*base.PullRequest, 0, len(resp.Repository.PullRequests.Nodes)) + // PR node id -> number, for the timeline node-id pass after this page + timelineTargets := map[string]int64{} + hitWatermark := false + for i := range resp.Repository.PullRequests.Nodes { + node := &resp.Repository.PullRequests.Nodes[i] + // Ordered DESC by updated_at: once we hit a PR older than the watermark, + // everything remaining is also older — stop processing this page and signal + // the caller to stop paging (no more new PRs to find). + if !since.IsZero() && node.UpdatedAt.Before(since) { + hitWatermark = true + break + } + + pr := g.convertGraphQLPullRequest(node) + allPRs = append(allPRs, pr) + timelineTargets[node.ID] = node.Number + // SECURITY: ensure the PR is safe (mirrors the REST path) + _ = CheckAndEnsureSafePR(pr, g.baseURL, g) + + // Cache the PR's issue-comments into the shared cache (served by the + // comment phase). Overflow -> REST for this PR's comments. + if node.Comments.TotalCount > len(node.Comments.Nodes) { + rest, err := g.getComments(ctx, pr) + if err != nil { + return nil, false, err + } + g.gqlComments[node.Number] = rest + } else { + g.gqlComments[node.Number] = convertGraphQLComments(node.Number, node.Comments.Nodes) + } + + // Cache reviews (with inline comments). If any review set or a review's + // comment set overflows one page, fall back to REST for the whole PR's + // reviews so nothing is dropped. + if g.reviewsOverflow(node) { + restReviews, err := g.GetReviews(ctx, pr) + if err != nil { + return nil, false, err + } + g.gqlReviews[node.Number] = restReviews + } else { + g.gqlReviews[node.Number] = convertGraphQLReviews(node) + } + } + + log.Info("metadata sync [%s/%s]: PRs page %d — %d new, %d timeline targets, watermark_reached=%v", + g.repoOwner, g.repoName, page, len(allPRs), len(timelineTargets), hitWatermark) + + // Best-effort: a timeline failure must not abort the PR/comment import (#37). + if len(timelineTargets) > 0 { + if err := g.attachTimelineEvents(ctx, timelineTargets); err != nil { + log.Error("github graphql: PR timeline events sync failed, importing without them: %v", err) + } + } + + // Stop paging when we've hit the watermark (no more new PRs) OR reached the last page. + done := hitWatermark || !resp.Repository.PullRequests.PageInfo.HasNextPage + return allPRs, done, nil +} + +func (g *GithubDownloaderV3) reviewsOverflow(node *gqlPullRequest) bool { + if node.Reviews.TotalCount > len(node.Reviews.Nodes) { + return true + } + for i := range node.Reviews.Nodes { + if node.Reviews.Nodes[i].Comments.TotalCount > len(node.Reviews.Nodes[i].Comments.Nodes) { + return true + } + } + return false +} + +func (g *GithubDownloaderV3) convertGraphQLPullRequest(node *gqlPullRequest) *base.PullRequest { + labels := make([]*base.Label, 0, len(node.Labels.Nodes)) + for _, l := range node.Labels.Nodes { + labels = append(labels, &base.Label{Name: l.Name, Color: l.Color, Description: l.Description}) + } + assignees := make([]string, 0, len(node.Assignees.Nodes)) + for _, a := range node.Assignees.Nodes { + assignees = append(assignees, a.Login) + } + var milestone string + if node.Milestone != nil { + milestone = node.Milestone.Title + } + var mergeCommitSHA string + if node.MergeCommit != nil { + mergeCommitSHA = node.MergeCommit.OID + } + + head := base.PullRequestBranch{Ref: node.HeadRefName, SHA: node.HeadRefOID} + if node.HeadRepo != nil { + head.RepoName = node.HeadRepo.Name + head.OwnerName = node.HeadRepo.Owner.Login + head.CloneURL = node.HeadRepo.URL + ".git" + } + baseBranch := base.PullRequestBranch{Ref: node.BaseRefName, SHA: node.BaseRefOID} + if node.BaseRepo != nil { + baseBranch.RepoName = node.BaseRepo.Name + baseBranch.OwnerName = node.BaseRepo.Owner.Login + } + + // GitHub REST reports a merged PR as state "closed"; keep that convention. + state := "open" + if node.State != "OPEN" { + state = "closed" + } + + return &base.PullRequest{ + Number: node.Number, + Title: node.Title, + PosterID: node.Author.DatabaseID, + PosterName: node.Author.Login, + Content: node.Body, + Milestone: milestone, + State: state, + Created: node.CreatedAt, + Updated: node.UpdatedAt, + Closed: node.ClosedAt, + Labels: labels, + // A raw patch download, not an API call — construct rather than resolve it. + PatchURL: fmt.Sprintf("%s/%s/%s/pull/%d.patch", g.baseURL, g.repoOwner, g.repoName, node.Number), + Merged: node.MergedAt != nil || node.State == "MERGED", + MergedTime: node.MergedAt, + MergeCommitSHA: mergeCommitSHA, + Head: head, + Base: baseBranch, + Assignees: assignees, + IsLocked: node.Locked, + Reactions: convertGraphQLReactions(node.Reactions), + ForeignIndex: node.Number, + IsDraft: node.IsDraft, + } +} + +// convertGraphQLReviews maps a PR's GraphQL reviews (and their inline comments) +// and its pending review requests onto base.Review, matching the REST path. +func convertGraphQLReviews(node *gqlPullRequest) []*base.Review { + reviews := make([]*base.Review, 0, len(node.Reviews.Nodes)+len(node.ReviewRequests.Nodes)) + for i := range node.Reviews.Nodes { + r := &node.Reviews.Nodes[i] + created := r.CreatedAt + if r.SubmittedAt != nil { + created = *r.SubmittedAt + } + var commitID string + if r.Commit != nil { + commitID = r.Commit.OID + } + reviews = append(reviews, &base.Review{ + ID: r.DatabaseID, + IssueIndex: node.Number, + ReviewerID: r.Author.DatabaseID, + ReviewerName: r.Author.Login, + CommitID: commitID, + Content: r.Body, + CreatedAt: created, + State: r.State, // GraphQL and REST share the review-state enum + Comments: convertGraphQLReviewComments(r), + }) + } + // Pending review requests become REQUEST_REVIEW pseudo-reviews (REST parity). + for _, rr := range node.ReviewRequests.Nodes { + if rr.RequestedReviewer.Login == "" { + continue // non-user reviewer (team) — TODO + } + reviews = append(reviews, &base.Review{ + IssueIndex: node.Number, + ReviewerID: rr.RequestedReviewer.DatabaseID, + ReviewerName: rr.RequestedReviewer.Login, + State: base.ReviewStateRequestReview, + }) + } + return reviews +} + +func convertGraphQLReviewComments(r *gqlReview) []*base.ReviewComment { + comments := make([]*base.ReviewComment, 0, len(r.Comments.Nodes)) + for i := range r.Comments.Nodes { + c := &r.Comments.Nodes[i] + var inReplyTo int64 + if c.ReplyTo != nil { + inReplyTo = c.ReplyTo.DatabaseID + } + var commitID string + if c.Commit != nil { + commitID = c.Commit.OID + } + comments = append(comments, &base.ReviewComment{ + ID: c.DatabaseID, + InReplyTo: inReplyTo, + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: c.Position, + CommitID: commitID, + PosterID: c.Author.DatabaseID, + CreatedAt: c.CreatedAt, + UpdatedAt: c.UpdatedAt, + }) + } + return comments +} diff --git a/services/migrations/github_graphql_pr_test.go b/services/migrations/github_graphql_pr_test.go new file mode 100644 index 0000000000000..a056430c5e207 --- /dev/null +++ b/services/migrations/github_graphql_pr_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "testing" + + "gitea.dev/modules/json" + base "gitea.dev/modules/migration" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGraphQLPullRequestParsing validates that the PR response structs match the +// shape of graphQLPullRequestsQuery and convert into the expected entities: +// pull request metadata + head/base, reviews, and inline review comments. +func TestGraphQLPullRequestParsing(t *testing.T) { + const sample = `{ + "repository": { + "pullRequests": { + "pageInfo": {"hasNextPage": false, "endCursor": "Y3Vyc29yOjIw"}, + "nodes": [ + { + "number": 7, + "title": "add feature", + "body": "does the thing", + "state": "MERGED", + "createdAt": "2023-05-01T00:00:00Z", + "updatedAt": "2023-06-01T00:00:00Z", + "closedAt": "2023-06-01T00:00:00Z", + "mergedAt": "2023-06-01T00:00:00Z", + "isDraft": false, + "locked": null, + "author": {"login": "contributor", "databaseId": 4242}, + "milestone": {"title": "v2"}, + "mergeCommit": {"oid": "abc123"}, + "headRefName": "feature-x", + "headRefOid": "deadbeef", + "baseRefName": "main", + "baseRefOid": "cafebabe", + "headRepository": {"name": "gitea", "url": "https://github.com/contributor/gitea", "owner": {"login": "contributor"}}, + "baseRepository": {"name": "gitea", "owner": {"login": "go-gitea"}}, + "labels": {"nodes": [{"name": "enhancement", "color": "a2eeef", "description": ""}]}, + "assignees": {"nodes": [{"login": "maintainer"}]}, + "reactions": {"nodes": [{"content": "ROCKET"}]}, + "comments": {"totalCount": 1, "nodes": [{"databaseId": 500, "body": "LGTM", "createdAt": "2023-05-02T00:00:00Z", "updatedAt": "2023-05-02T00:00:00Z", "author": {"login": "x", "databaseId": 1}, "reactions": {"nodes": []}}]}, + "reviews": {"totalCount": 1, "nodes": [ + {"databaseId": 900, "state": "CHANGES_REQUESTED", "body": "please fix", "createdAt": "2023-05-03T00:00:00Z", "submittedAt": "2023-05-03T01:00:00Z", "author": {"login": "reviewer", "databaseId": 77}, "commit": {"oid": "sha1sha"}, "comments": {"totalCount": 1, "nodes": [ + {"databaseId": 950, "body": "nit here", "path": "main.go", "diffHunk": "@@ -1 +1 @@", "position": 3, "commit": {"oid": "sha1sha"}, "author": {"login": "reviewer", "databaseId": 77}, "createdAt": "2023-05-03T01:00:00Z", "updatedAt": "2023-05-03T01:00:00Z", "replyTo": null} + ]}} + ]}, + "reviewRequests": {"nodes": [{"requestedReviewer": {"login": "pending-rev", "databaseId": 88}}]} + } + ] + } + }, + "rateLimit": {"cost": 12, "remaining": 4988, "resetAt": "2023-01-01T00:00:00Z"} + }` + + var resp gqlPullRequestsResponse + require.NoError(t, json.Unmarshal([]byte(sample), &resp)) + require.Len(t, resp.Repository.PullRequests.Nodes, 1) + assert.False(t, resp.Repository.PullRequests.PageInfo.HasNextPage) + + node := &resp.Repository.PullRequests.Nodes[0] + g := &GithubDownloaderV3{baseURL: "https://github.com", repoOwner: "go-gitea", repoName: "gitea"} + + // PR metadata + head/base + constructed patch URL + merged/closed convention + pr := g.convertGraphQLPullRequest(node) + assert.EqualValues(t, 7, pr.Number) + assert.Equal(t, "closed", pr.State) // MERGED reports as closed, like REST + assert.True(t, pr.Merged) + assert.Equal(t, "abc123", pr.MergeCommitSHA) + assert.Equal(t, "https://github.com/go-gitea/gitea/pull/7.patch", pr.PatchURL) + assert.Equal(t, "feature-x", pr.Head.Ref) + assert.Equal(t, "deadbeef", pr.Head.SHA) + assert.Equal(t, "contributor", pr.Head.OwnerName) + assert.Equal(t, "https://github.com/contributor/gitea.git", pr.Head.CloneURL) + assert.Equal(t, "main", pr.Base.Ref) + assert.Equal(t, "go-gitea", pr.Base.OwnerName) + require.Len(t, pr.Reactions, 1) + assert.Equal(t, "rocket", pr.Reactions[0].Content) + + // reviews: one real review (with an inline comment) + one requested-reviewer + assert.False(t, g.reviewsOverflow(node)) + reviews := convertGraphQLReviews(node) + require.Len(t, reviews, 2) + assert.EqualValues(t, 900, reviews[0].ID) + assert.Equal(t, "CHANGES_REQUESTED", reviews[0].State) + assert.EqualValues(t, 7, reviews[0].IssueIndex) + require.Len(t, reviews[0].Comments, 1) + assert.EqualValues(t, 950, reviews[0].Comments[0].ID) + assert.Equal(t, "main.go", reviews[0].Comments[0].TreePath) + assert.Equal(t, "sha1sha", reviews[0].Comments[0].CommitID) + // the pending review request + assert.Equal(t, base.ReviewStateRequestReview, reviews[1].State) + assert.Equal(t, "pending-rev", reviews[1].ReviewerName) +} + +// TestGraphQLReviewsOverflow verifies the REST-fallback trigger when a PR has +// more reviews (or a review has more inline comments) than one page returned. +func TestGraphQLReviewsOverflow(t *testing.T) { + g := &GithubDownloaderV3{} + // reviews truncated: totalCount > returned nodes + over := &gqlPullRequest{} + over.Reviews.TotalCount = 60 + over.Reviews.Nodes = make([]gqlReview, 50) + assert.True(t, g.reviewsOverflow(over)) + + // a review's inline comments truncated + inner := &gqlPullRequest{} + inner.Reviews.TotalCount = 1 + inner.Reviews.Nodes = make([]gqlReview, 1) + inner.Reviews.Nodes[0].Comments.TotalCount = 80 + inner.Reviews.Nodes[0].Comments.Nodes = make([]gqlReviewComment, 50) + assert.True(t, g.reviewsOverflow(inner)) + + // fully contained: no overflow + ok := &gqlPullRequest{} + ok.Reviews.TotalCount = 2 + ok.Reviews.Nodes = make([]gqlReview, 2) + assert.False(t, g.reviewsOverflow(ok)) +} diff --git a/services/migrations/github_graphql_test.go b/services/migrations/github_graphql_test.go new file mode 100644 index 0000000000000..a5ce8c3248033 --- /dev/null +++ b/services/migrations/github_graphql_test.go @@ -0,0 +1,218 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "gitea.dev/modules/json" + base "gitea.dev/modules/migration" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsGraphQLRateLimited(t *testing.T) { + assert.True(t, isGraphQLRateLimited([]graphQLError{{Type: "RATE_LIMITED", Message: "exceeded"}})) + assert.True(t, isGraphQLRateLimited([]graphQLError{{Type: "NOT_FOUND"}, {Type: "RATE_LIMITED"}})) + assert.False(t, isGraphQLRateLimited([]graphQLError{{Type: "NOT_FOUND"}})) + assert.False(t, isGraphQLRateLimited(nil)) +} + +func TestGraphQLRateResetWait(t *testing.T) { + // Retry-After (seconds) takes precedence + h := http.Header{} + h.Set("Retry-After", "30") + h.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)) + assert.Equal(t, 30*time.Second, graphQLRateResetWait(h)) + + // X-RateLimit-Reset (unix epoch, in the future) when no Retry-After + h = http.Header{} + h.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(90*time.Second).Unix(), 10)) + got := graphQLRateResetWait(h) + assert.Greater(t, got, 60*time.Second) + assert.LessOrEqual(t, got, 90*time.Second) + + // a reset already in the past, and no headers at all, both fall back + past := http.Header{} + past.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(-time.Hour).Unix(), 10)) + assert.Equal(t, graphQLRateResetFallback, graphQLRateResetWait(past)) + assert.Equal(t, graphQLRateResetFallback, graphQLRateResetWait(http.Header{})) + + // a bogus far-future reset is capped + huge := http.Header{} + huge.Set("Retry-After", "999999") + assert.Equal(t, graphQLRateResetCap, graphQLRateResetWait(huge)) +} + +// TestDoGraphQLRetriesOnRateLimited proves the fix for the abort-on-RATE_LIMITED +// bug: a RATE_LIMITED response is waited out (per the Retry-After header) and the +// request retried in place, instead of erroring and aborting the whole sync. +func TestDoGraphQLRetriesOnRateLimited(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.Header().Set("Retry-After", "1") // 1s so the test stays fast + io.WriteString(w, `{"errors":[{"type":"RATE_LIMITED","message":"API rate limit exceeded"}]}`) + return + } + io.WriteString(w, `{"data":{"ok":true}}`) + })) + defer srv.Close() + + d, err := NewGithubDownloaderV3(t.Context(), srv.URL, "", "", "tok", "o", "r") + require.NoError(t, err) + + var out struct { + OK bool `json:"ok"` + } + require.NoError(t, d.doGraphQL(t.Context(), "query{}", nil, &out)) + assert.True(t, out.OK) + assert.EqualValues(t, 2, calls.Load(), "RATE_LIMITED should be waited out and the request retried once") +} + +// TestGraphQLQueryReactionGating guards the MAX_NODE_LIMIT_EXCEEDED fix: reactions +// must never be nested under comments (issues×comments×reactions = 1,000,000 nodes, +// which GitHub rejects), and must be omitted entirely when SkipReactions is set. +// The reactions COUNT is the tell — with reactions wanted it must appear exactly +// once (the entity level); a second occurrence means the fatal comment-nesting +// regressed. +func TestGraphQLQueryReactionGating(t *testing.T) { + for _, tc := range []struct { + name string + query func(*GithubDownloaderV3) string + }{ + {"issues", (*GithubDownloaderV3).graphQLIssuesQuery}, + {"pulls", (*GithubDownloaderV3).graphQLPullRequestsQuery}, + } { + skip := tc.query(&GithubDownloaderV3{SkipReactions: true}) + keep := tc.query(&GithubDownloaderV3{SkipReactions: false}) + assert.Equal(t, 0, strings.Count(skip, "reactions"), "%s: SkipReactions must omit all reactions", tc.name) + assert.Equal(t, 1, strings.Count(keep, "reactions"), "%s: reactions only at the entity level, never nested under comments", tc.name) + // the comments connection must carry no nested reactions in either mode + assert.NotContains(t, skip, "comments(first:100){totalCount nodes{databaseId body createdAt updatedAt author{login ... on User{databaseId}} reactions", tc.name) + } +} + +func TestGraphQLEndpoint(t *testing.T) { + assert.Equal(t, "https://api.github.com/graphql", (&GithubDownloaderV3{baseURL: "https://github.com"}).graphQLEndpoint()) + assert.Equal(t, "https://api.github.com/graphql", (&GithubDownloaderV3{baseURL: ""}).graphQLEndpoint()) + assert.Equal(t, "https://ghe.example.com/api/graphql", (&GithubDownloaderV3{baseURL: "https://ghe.example.com"}).graphQLEndpoint()) + assert.Equal(t, "https://ghe.example.com/api/graphql", (&GithubDownloaderV3{baseURL: "https://ghe.example.com/"}).graphQLEndpoint()) +} + +func TestGraphQLReactionContent(t *testing.T) { + assert.Equal(t, "+1", graphQLReactionContent("THUMBS_UP")) + assert.Equal(t, "-1", graphQLReactionContent("THUMBS_DOWN")) + assert.Equal(t, "heart", graphQLReactionContent("HEART")) + assert.Equal(t, "rocket", graphQLReactionContent("ROCKET")) + // unknown enum falls back to lower-case + assert.Equal(t, "star", graphQLReactionContent("STAR")) +} + +// TestGraphQLIssuesResponseParsing validates that the response structs match the +// shape of graphQLIssuesQuery and that a page parses into the expected entities. +func TestGraphQLIssuesResponseParsing(t *testing.T) { + const sample = `{ + "repository": { + "issues": { + "pageInfo": {"hasNextPage": true, "endCursor": "Y3Vyc29yOjEwMA=="}, + "nodes": [ + { + "number": 42, + "title": "a bug", + "body": "it broke", + "state": "CLOSED", + "createdAt": "2024-01-02T03:04:05Z", + "updatedAt": "2024-02-03T04:05:06Z", + "closedAt": "2024-02-03T04:05:06Z", + "locked": true, + "author": {"login": "octocat", "databaseId": 583231}, + "milestone": {"title": "v1.0"}, + "labels": {"nodes": [{"name": "bug", "color": "d73a4a", "description": "defect"}]}, + "assignees": {"nodes": [{"login": "maintainer"}]}, + "reactions": {"nodes": [{"content": "THUMBS_UP"}, {"content": "HEART"}]}, + "comments": { + "totalCount": 1, + "nodes": [ + {"databaseId": 900001, "body": "same here", "createdAt": "2024-01-05T00:00:00Z", "updatedAt": "2024-01-05T00:00:00Z", "author": {"login": "reporter", "databaseId": 111}, "reactions": {"nodes": [{"content": "ROCKET"}]}} + ] + } + } + ] + } + }, + "rateLimit": {"cost": 1, "remaining": 4999, "resetAt": "2024-01-01T00:00:00Z"} + }` + + var resp gqlIssuesResponse + require.NoError(t, json.Unmarshal([]byte(sample), &resp)) + + assert.True(t, resp.Repository.Issues.PageInfo.HasNextPage) + assert.Equal(t, "Y3Vyc29yOjEwMA==", resp.Repository.Issues.PageInfo.EndCursor) + assert.Equal(t, 4999, resp.RateLimit.Remaining) + require.Len(t, resp.Repository.Issues.Nodes, 1) + + node := resp.Repository.Issues.Nodes[0] + assert.EqualValues(t, 42, node.Number) + assert.Equal(t, "CLOSED", node.State) + assert.Equal(t, "octocat", node.Author.Login) + assert.EqualValues(t, 583231, node.Author.DatabaseID) + assert.True(t, node.Locked) + require.NotNil(t, node.Milestone) + assert.Equal(t, "v1.0", node.Milestone.Title) + + // reactions convert with the enum→content mapping + reactions := convertGraphQLReactions(node.Reactions) + require.Len(t, reactions, 2) + assert.Equal(t, "+1", reactions[0].Content) + assert.Equal(t, "heart", reactions[1].Content) + + // comments convert and carry the issue number as IssueIndex + remote id as Index + comments := convertGraphQLComments(node.Number, node.Comments.Nodes) + require.Len(t, comments, 1) + assert.EqualValues(t, 42, comments[0].IssueIndex) + assert.EqualValues(t, 900001, comments[0].Index) + assert.Equal(t, "reporter", comments[0].PosterName) + require.Len(t, comments[0].Reactions, 1) + assert.Equal(t, "rocket", comments[0].Reactions[0].Content) +} + +// TestGetCachedComments checks the cache-paging that serves GraphQL-fetched +// comments to the framework's comment phase. +func TestGetCachedComments(t *testing.T) { + g := &GithubDownloaderV3{ + useGraphQL: true, + gqlComments: map[int64][]*base.Comment{ + 2: {{IssueIndex: 2, Index: 20}, {IssueIndex: 2, Index: 21}}, + 1: {{IssueIndex: 1, Index: 10}}, + }, + } + // page 1, size 2: ordered by issue number (1 then 2) + page1, isEnd, err := g.getCachedComments(1, 2) + require.NoError(t, err) + assert.False(t, isEnd) + require.Len(t, page1, 2) + assert.EqualValues(t, 10, page1[0].Index) + assert.EqualValues(t, 20, page1[1].Index) + + page2, isEnd, err := g.getCachedComments(2, 2) + require.NoError(t, err) + assert.True(t, isEnd) + require.Len(t, page2, 1) + assert.EqualValues(t, 21, page2[0].Index) + + // past the end returns empty + end + page3, isEnd, err := g.getCachedComments(3, 2) + require.NoError(t, err) + assert.True(t, isEnd) + assert.Empty(t, page3) +} diff --git a/services/migrations/github_graphql_timeline.go b/services/migrations/github_graphql_timeline.go new file mode 100644 index 0000000000000..672ac77a0b95c --- /dev/null +++ b/services/migrations/github_graphql_timeline.go @@ -0,0 +1,318 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +// Issue/PR timeline events (closed, reopened, locked, unlocked, renamed, labeled, +// milestoned) are imported as typed Gitea comments so the mirror shows issue +// history — who closed/labeled/locked and when — not just the final state. +// +// They are fetched by a batched node-id pass (like reactions, #27), NOT nested in +// the content query: a 3rd connection level under issues/PRs blows GitHub's +// per-query compute budget (RESOURCE_LIMITS_EXCEEDED, #27). Unlike reactions, +// timeline events bump the issue's updated_at, so a new event re-surfaces the issue +// in the normal watermark sync — no periodic rescan — and they are immutable and +// append-only, so no removal reconcile. Gitea renders the timeline sorted by +// created_unix across comments and typed events, so fetch order is irrelevant as +// long as each event carries its real event timestamp. +// +// Assignee events are intentionally omitted: a Gitea assignee comment references a +// user by id with no name fallback, and migrated GitHub users are not Gitea users. + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "maps" + "slices" + "time" + + "gitea.dev/modules/log" + base "gitea.dev/modules/migration" +) + +// timelineBatchSize is how many issue/PR node ids are asked for their timeline per +// request. +const timelineBatchSize = 100 + +// Timeline events must be split by union: MERGED/HEAD_REF_DELETED/AUTO_MERGE_* are +// PullRequest-only — including them in the Issue timelineItems (itemTypes arg OR an +// `... on MergedEvent` fragment) is a hard GraphQL validation error that aborts the +// whole timeline pass. issueTimelineItemTypes is the set valid on BOTH unions; the PR +// query adds the PR-only types on top. +const ( + issueTimelineItemTypes = "CLOSED_EVENT,REOPENED_EVENT,LOCKED_EVENT,UNLOCKED_EVENT,RENAMED_TITLE_EVENT,LABELED_EVENT,UNLABELED_EVENT,MILESTONED_EVENT,DEMILESTONED_EVENT,PINNED_EVENT,UNPINNED_EVENT" + prTimelineItemTypes = issueTimelineItemTypes + ",MERGED_EVENT,HEAD_REF_DELETED_EVENT,AUTO_MERGE_ENABLED_EVENT,AUTO_MERGE_DISABLED_EVENT" +) + +// Inline-fragment selections, likewise split. These event types expose no databaseId, +// only the node id. +const issueTimelineFragments = ` + ... on ClosedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on ReopenedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on LockedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on UnlockedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on RenamedTitleEvent{id createdAt previousTitle currentTitle actor{login ... on User{databaseId}}} + ... on LabeledEvent{id createdAt label{name} actor{login ... on User{databaseId}}} + ... on UnlabeledEvent{id createdAt label{name} actor{login ... on User{databaseId}}} + ... on MilestonedEvent{id createdAt milestoneTitle actor{login ... on User{databaseId}}} + ... on DemilestonedEvent{id createdAt milestoneTitle actor{login ... on User{databaseId}}} + ... on PinnedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on UnpinnedEvent{id createdAt actor{login ... on User{databaseId}}}` + +const prTimelineFragments = issueTimelineFragments + ` + ... on MergedEvent{id createdAt actor{login ... on User{databaseId}}} + ... on HeadRefDeletedEvent{id createdAt headRefName actor{login ... on User{databaseId}}} + ... on AutoMergeEnabledEvent{id createdAt actor{login ... on User{databaseId}}} + ... on AutoMergeDisabledEvent{id createdAt actor{login ... on User{databaseId}}}` + +// timelineConn wraps a fragment selection in the shared connection envelope. +func timelineConn(fragments string) string { + return `{ + pageInfo{hasNextPage endCursor} + nodes{ + __typename` + fragments + ` + } + }` +} + +// graphQLTimelineBatchQuery fetches the first page of timeline events for a batch of +// issue/PR node ids in one request (two levels deep). Issue and PullRequest both +// expose timelineItems, and the response merges to the same `timelineItems` key. +func graphQLTimelineBatchQuery() string { + return fmt.Sprintf(` +query($ids:[ID!]!){ + nodes(ids:$ids){ + id + ... on Issue{timelineItems(first:100,itemTypes:[%[1]s])%[2]s} + ... on PullRequest{timelineItems(first:100,itemTypes:[%[3]s])%[4]s} + } + rateLimit{cost remaining resetAt} +}`, issueTimelineItemTypes, timelineConn(issueTimelineFragments), prTimelineItemTypes, timelineConn(prTimelineFragments)) +} + +// graphQLTimelineSweepQuery pages the remaining timeline of a single hot issue/PR +// (one with more than one page of events), by node id. +func graphQLTimelineSweepQuery() string { + return fmt.Sprintf(` +query($id:ID!,$cursor:String){ + node(id:$id){ + ... on Issue{timelineItems(first:100,after:$cursor,itemTypes:[%[1]s])%[2]s} + ... on PullRequest{timelineItems(first:100,after:$cursor,itemTypes:[%[3]s])%[4]s} + } + rateLimit{cost remaining resetAt} +}`, issueTimelineItemTypes, timelineConn(issueTimelineFragments), prTimelineItemTypes, timelineConn(prTimelineFragments)) +} + +type gqlTimelineItem struct { + Typename string `json:"__typename"` + ID string `json:"id"` + CreatedAt time.Time `json:"createdAt"` + Actor gqlActor `json:"actor"` + PreviousTitle string `json:"previousTitle"` + CurrentTitle string `json:"currentTitle"` + Label *struct { + Name string `json:"name"` + } `json:"label"` + MilestoneTitle string `json:"milestoneTitle"` + HeadRefName string `json:"headRefName"` +} + +type gqlTimelineConn struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + Nodes []gqlTimelineItem `json:"nodes"` +} + +// timelineEventID hashes a timeline event's GraphQL node id (these events carry no +// databaseId) to a stable positive int64 for the OriginalID dedup key, so a re-sync +// upserts each event rather than duplicating it. +func timelineEventID(nodeID string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(nodeID)) + return int64(h.Sum64() & 0x7FFFFFFFFFFFFFFF) +} + +// convertTimelineItem maps one timeline event to a typed base.Comment. Returns nil +// for events that carry no usable payload (e.g. a label event with no label). +func convertTimelineItem(it *gqlTimelineItem) *base.Comment { + c := &base.Comment{ + Index: timelineEventID(it.ID), + PosterID: it.Actor.DatabaseID, + PosterName: it.Actor.Login, + Created: it.CreatedAt, + Updated: it.CreatedAt, + } + switch it.Typename { + case "ClosedEvent": + c.CommentType = "close" + case "ReopenedEvent": + c.CommentType = "reopen" + case "LockedEvent": + c.CommentType = "lock" + case "UnlockedEvent": + c.CommentType = "unlock" + case "RenamedTitleEvent": + c.CommentType = "change_title" + c.Meta = map[string]any{"OldTitle": it.PreviousTitle, "NewTitle": it.CurrentTitle} + case "LabeledEvent": + if it.Label == nil { + return nil + } + c.CommentType = "label" + c.Content = "1" // Gitea marks an add with Content "1", a remove with "" + c.Meta = map[string]any{"LabelName": it.Label.Name} + case "UnlabeledEvent": + if it.Label == nil { + return nil + } + c.CommentType = "label" + c.Meta = map[string]any{"LabelName": it.Label.Name} + case "MilestonedEvent": + c.CommentType = "milestone" + c.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle} + case "DemilestonedEvent": + c.CommentType = "milestone" + c.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle, "Removed": true} + case "MergedEvent": + c.CommentType = "merge_pull" + case "HeadRefDeletedEvent": + c.CommentType = "delete_branch" + c.Content = it.HeadRefName + case "PinnedEvent": + c.CommentType = "pin" + case "UnpinnedEvent": + c.CommentType = "unpin" + case "AutoMergeEnabledEvent": + c.CommentType = "pull_scheduled_merge" + case "AutoMergeDisabledEvent": + c.CommentType = "pull_cancel_scheduled_merge" + default: + return nil + } + return c +} + +func convertTimelineItems(items []gqlTimelineItem) []*base.Comment { + comments := make([]*base.Comment, 0, len(items)) + for i := range items { + if c := convertTimelineItem(&items[i]); c != nil { + comments = append(comments, c) + } + } + return comments +} + +// attachTimelineEvents fetches timeline events for the given issues/PRs (keyed by +// GraphQL node id → local index) and appends them as typed comments to the shared +// comment cache, so the comment phase persists them interleaved with regular +// comments. No-op when the map is empty. +func (g *GithubDownloaderV3) attachTimelineEvents(ctx context.Context, nodeIDToIndex map[string]int64) error { + if len(nodeIDToIndex) == 0 { + return nil + } + log.Info("metadata sync [%s/%s]: timeline — fetching events for %d issues/PRs", + g.repoOwner, g.repoName, len(nodeIDToIndex)) + ids := make([]string, 0, len(nodeIDToIndex)) + for id := range nodeIDToIndex { + if id != "" { + ids = append(ids, id) + } + } + slices.Sort(ids) // deterministic batching + for start := 0; start < len(ids); start += timelineBatchSize { + end := min(start+timelineBatchSize, len(ids)) + eventsByID, err := g.fetchTimelineEvents(ctx, ids[start:end]) + if err != nil { + return err + } + for id, events := range eventsByID { + index := nodeIDToIndex[id] + for _, e := range events { + e.IssueIndex = index + } + g.gqlComments[index] = append(g.gqlComments[index], events...) + } + } + return nil +} + +// fetchTimelineEvents returns timeline events for a batch of issue/PR node ids. On +// RESOURCE_LIMITS_EXCEEDED it halves the batch and retries (adaptive shrink) rather +// than aborting; a hot entity with more than one page sweeps the remainder. +func (g *GithubDownloaderV3) fetchTimelineEvents(ctx context.Context, ids []string) (map[string][]*base.Comment, error) { + if len(ids) == 0 { + return map[string][]*base.Comment{}, nil + } + var resp struct { + Nodes []struct { + ID string `json:"id"` + TimelineItems gqlTimelineConn `json:"timelineItems"` + } `json:"nodes"` + RateLimit graphQLRateLimit `json:"rateLimit"` + } + if err := g.doGraphQL(ctx, graphQLTimelineBatchQuery(), map[string]any{"ids": ids}, &resp); err != nil { + var ge graphQLError + if errors.As(err, &ge) && ge.Type == "RESOURCE_LIMITS_EXCEEDED" && len(ids) > 1 { + mid := len(ids) / 2 + left, err := g.fetchTimelineEvents(ctx, ids[:mid]) + if err != nil { + return nil, err + } + right, err := g.fetchTimelineEvents(ctx, ids[mid:]) + if err != nil { + return nil, err + } + maps.Copy(left, right) + + return left, nil + } + return nil, err + } + g.respectGraphQLBudget(ctx, resp.RateLimit) + + out := make(map[string][]*base.Comment, len(resp.Nodes)) + for i := range resp.Nodes { + n := &resp.Nodes[i] + if n.ID == "" { + continue // null node (unresolvable id) + } + events := convertTimelineItems(n.TimelineItems.Nodes) + if n.TimelineItems.PageInfo.HasNextPage { + rest, err := g.sweepTimelineEvents(ctx, n.ID, n.TimelineItems.PageInfo.EndCursor) + if err != nil { + return nil, err + } + events = append(events, rest...) + } + out[n.ID] = events + } + return out, nil +} + +// sweepTimelineEvents pages the remaining timeline of one node over GraphQL, +// following the cursor. +func (g *GithubDownloaderV3) sweepTimelineEvents(ctx context.Context, nodeID, cursor string) ([]*base.Comment, error) { + var all []*base.Comment + for { + vars := map[string]any{"id": nodeID, "cursor": cursor} + var resp struct { + Node struct { + TimelineItems gqlTimelineConn `json:"timelineItems"` + } `json:"node"` + RateLimit graphQLRateLimit `json:"rateLimit"` + } + if err := g.doGraphQL(ctx, graphQLTimelineSweepQuery(), vars, &resp); err != nil { + return nil, err + } + g.respectGraphQLBudget(ctx, resp.RateLimit) + all = append(all, convertTimelineItems(resp.Node.TimelineItems.Nodes)...) + if !resp.Node.TimelineItems.PageInfo.HasNextPage { + return all, nil + } + cursor = resp.Node.TimelineItems.PageInfo.EndCursor + } +} diff --git a/services/migrations/github_graphql_timeline_test.go b/services/migrations/github_graphql_timeline_test.go new file mode 100644 index 0000000000000..5d10c2a8d9d80 --- /dev/null +++ b/services/migrations/github_graphql_timeline_test.go @@ -0,0 +1,123 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "strings" + "testing" + + "gitea.dev/modules/json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestConvertTimelineItems parses a timeline page (the shape of #13603's real +// timeline plus milestone/rename) and checks each event maps to the right Gitea +// comment type, meta, actor, and a stable dedup id — and that unmapped types are +// dropped. +func TestConvertTimelineItems(t *testing.T) { + const sample = `{"pageInfo":{"hasNextPage":false,"endCursor":""},"nodes":[ + {"__typename":"ClosedEvent","id":"c1","createdAt":"2020-11-17T15:54:33Z","actor":{"login":"jolheiser","databaseId":42128690}}, + {"__typename":"LabeledEvent","id":"l1","createdAt":"2020-11-17T15:54:38Z","label":{"name":"type/question"},"actor":{"login":"jolheiser"}}, + {"__typename":"UnlabeledEvent","id":"u1","createdAt":"2020-11-18T00:00:00Z","label":{"name":"type/bug"},"actor":{"login":"jolheiser"}}, + {"__typename":"LockedEvent","id":"k1","createdAt":"2020-11-17T16:31:00Z","actor":{"login":"go-gitea"}}, + {"__typename":"RenamedTitleEvent","id":"r1","createdAt":"2020-11-19T00:00:00Z","previousTitle":"old","currentTitle":"new","actor":{"login":"x"}}, + {"__typename":"MilestonedEvent","id":"m1","createdAt":"2020-11-20T00:00:00Z","milestoneTitle":"v1.14","actor":{"login":"y"}}, + {"__typename":"DemilestonedEvent","id":"d1","createdAt":"2020-11-21T00:00:00Z","milestoneTitle":"v1.13","actor":{"login":"z"}}, + {"__typename":"MergedEvent","id":"mg1","createdAt":"2020-11-22T00:00:00Z","actor":{"login":"a"}}, + {"__typename":"HeadRefDeletedEvent","id":"hd1","createdAt":"2020-11-22T01:00:00Z","headRefName":"feature/x","actor":{"login":"b"}}, + {"__typename":"PinnedEvent","id":"pn1","createdAt":"2020-11-22T02:00:00Z","actor":{"login":"c"}}, + {"__typename":"UnpinnedEvent","id":"up1","createdAt":"2020-11-22T03:00:00Z","actor":{"login":"d"}}, + {"__typename":"AutoMergeEnabledEvent","id":"am1","createdAt":"2020-11-22T04:00:00Z","actor":{"login":"e"}}, + {"__typename":"AutoMergeDisabledEvent","id":"ad1","createdAt":"2020-11-22T05:00:00Z","actor":{"login":"f"}}, + {"__typename":"MentionedEvent","id":"skip"} + ]}` + var conn gqlTimelineConn + require.NoError(t, json.Unmarshal([]byte(sample), &conn)) + events := convertTimelineItems(conn.Nodes) + require.Len(t, events, 13) // MentionedEvent dropped + + assert.Equal(t, "close", events[0].CommentType) + assert.Equal(t, "jolheiser", events[0].PosterName) + assert.EqualValues(t, 42128690, events[0].PosterID) + + assert.Equal(t, "label", events[1].CommentType) + assert.Equal(t, "1", events[1].Content) // add + assert.Equal(t, "type/question", events[1].Meta["LabelName"]) + + assert.Equal(t, "label", events[2].CommentType) + assert.Empty(t, events[2].Content) // remove + assert.Equal(t, "type/bug", events[2].Meta["LabelName"]) + + assert.Equal(t, "lock", events[3].CommentType) + + assert.Equal(t, "change_title", events[4].CommentType) + assert.Equal(t, "old", events[4].Meta["OldTitle"]) + assert.Equal(t, "new", events[4].Meta["NewTitle"]) + + assert.Equal(t, "milestone", events[5].CommentType) + assert.Equal(t, "v1.14", events[5].Meta["MilestoneTitle"]) + assert.Nil(t, events[5].Meta["Removed"]) + + assert.Equal(t, "milestone", events[6].CommentType) + assert.Equal(t, true, events[6].Meta["Removed"]) + + assert.Equal(t, "merge_pull", events[7].CommentType) + assert.Equal(t, "delete_branch", events[8].CommentType) + assert.Equal(t, "feature/x", events[8].Content) // deleted branch name + assert.Equal(t, "pin", events[9].CommentType) + assert.Equal(t, "unpin", events[10].CommentType) + assert.Equal(t, "pull_scheduled_merge", events[11].CommentType) + assert.Equal(t, "pull_cancel_scheduled_merge", events[12].CommentType) + + // every event carries its real timestamp and a stable positive dedup id + for _, e := range events { + assert.False(t, e.Created.IsZero(), "event must carry its GitHub timestamp") + assert.Positive(t, e.Index, "event must carry a stable dedup id") + } +} + +func TestTimelineEventIDStable(t *testing.T) { + id := "MDExOkNsb3NlZEV2ZW50NDAwNjIwMjg0Mw==" + idVal := timelineEventID(id) + assert.Equal(t, idVal, timelineEventID(id), "same node id -> same dedup key (idempotent re-sync)") + assert.NotEqual(t, timelineEventID(id), timelineEventID("MDEyOkxhYmVsZWRFdmVudA==")) + assert.Positive(t, timelineEventID(id)) +} + +// TestTimelineQueryShape guards the fix for #27's class of bug: the timeline query +// is a node-id pass (≤2 levels), never nested in the content query, and carries no +// reactions. +func TestTimelineQueryShape(t *testing.T) { + batch := graphQLTimelineBatchQuery() + assert.Contains(t, batch, "nodes(ids:$ids)") + assert.Contains(t, batch, "... on Issue{timelineItems") + assert.Contains(t, batch, "... on PullRequest{timelineItems") + assert.Contains(t, batch, "CLOSED_EVENT") + assert.Contains(t, batch, "LABELED_EVENT") + assert.Contains(t, batch, "MILESTONED_EVENT") + assert.Contains(t, batch, "... on ClosedEvent") + assert.NotContains(t, batch, "reactions", "timeline query must not carry reactions") + + sweep := graphQLTimelineSweepQuery() + assert.Contains(t, sweep, "node(id:$id)") + assert.Contains(t, sweep, "after:$cursor") + + // The timeline queries must be syntactically valid — brace-balanced. A missing + // close in timelineConnFields made the whole query malformed (Expected NAME at + // [35,1]), which aborted the entire metadata import (#37). + for name, q := range map[string]string{"batch": batch, "sweep": sweep} { + assert.Equal(t, strings.Count(q, "{"), strings.Count(q, "}"), "%s timeline query must be brace-balanced", name) + } +} + +// TestGraphQLQueriesFetchNodeID guards #28: the issue/PR node id must be selected, +// or node.ID is empty, the timeline (and reaction) node-id passes never run, and +// the feature silently no-ops. +func TestGraphQLQueriesFetchNodeID(t *testing.T) { + g := &GithubDownloaderV3{} + assert.Contains(t, g.graphQLIssuesQuery(), "\n id number ", "issue query must select the node id") + assert.Contains(t, g.graphQLPullRequestsQuery(), "\n id number ", "PR query must select the node id") +} diff --git a/services/migrations/http_client.go b/services/migrations/http_client.go index 6fe7440c55b84..1a0c805ab1b91 100644 --- a/services/migrations/http_client.go +++ b/services/migrations/http_client.go @@ -4,10 +4,16 @@ package migrations import ( + "bytes" "crypto/tls" + "io" "net/http" + "strconv" + "strings" + "time" "gitea.dev/modules/hostmatcher" + "gitea.dev/modules/log" "gitea.dev/modules/proxy" "gitea.dev/modules/setting" "gitea.dev/modules/util" @@ -20,10 +26,15 @@ import ( // concurrent callers sharing a single client instead of racing to create their own. var migrationHTTPClient = util.OnceValue[*http.Client]{Func: newMigrationHTTPClient} +// NewMigrationHTTPClient returns a new HTTP client for migration with retry support +func NewMigrationHTTPClient() *http.Client { + return newMigrationHTTPClient() +} + // newMigrationHTTPClient returns a HTTP client for migration func newMigrationHTTPClient() *http.Client { return &http.Client{ - Transport: NewMigrationHTTPTransport(), + Transport: newRetryTransport(NewMigrationHTTPTransport()), } } @@ -40,3 +51,118 @@ func NewMigrationHTTPTransport() *http.Transport { return hostmatcher.NewHTTPTransport("migration", allowList, blockList, proxy.Proxy(), setting.Proxy.ProxyURLFixed, &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}) } + +const ( + retryMaxRetries = 5 + retryBaseDelay = time.Second + retryMaxDelay = 30 * time.Second + // cap an honored Retry-After so a hostile/huge value can't stall a sync + retryMaxRetryAfter = 5 * time.Minute +) + +// retryTransport wraps an http.RoundTripper and transparently retries migration +// API requests that fail with transient errors: network failures and 5xx +// responses (e.g. a 502/503/504 from GitHub), plus secondary-rate-limit +// responses (403/429) that carry a Retry-After. Without this, a single transient +// hiccup — such as a 504 while fetching one issue's reactions midway through a +// large repository's metadata sweep — aborts the entire sync. Primary rate limit +// (403 with X-RateLimit-Remaining: 0 and no Retry-After) is intentionally NOT +// retried here; the downloader already waits for the reset window before its +// calls, so we leave that handling to it. +type retryTransport struct { + base http.RoundTripper + maxRetries int + baseDelay time.Duration +} + +func newRetryTransport(base http.RoundTripper) http.RoundTripper { + return &retryTransport{base: base, maxRetries: retryMaxRetries, baseDelay: retryBaseDelay} +} + +func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // Buffer the body so the request can be replayed on each retry. + var bodyBytes []byte + if req.Body != nil { + var err error + bodyBytes, err = io.ReadAll(req.Body) + _ = req.Body.Close() + if err != nil { + return nil, err + } + } + + delay := t.baseDelay + for attempt := 0; ; attempt++ { + if bodyBytes != nil { + req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + } + + resp, err := t.base.RoundTrip(req) + if attempt >= t.maxRetries || !shouldRetryRequest(resp, err) { + return resp, err + } + + wait := delay + if resp != nil { + if ra := parseRetryAfter(resp.Header.Get("Retry-After")); ra > 0 { + wait = min(ra, retryMaxRetryAfter) + } + // drain and close so the underlying connection can be reused + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + + log.Warn("migration: transient request failure (%s), retry %d/%d in %s", retryReason(resp, err), attempt+1, t.maxRetries, wait) + + timer := time.NewTimer(wait) + select { + case <-req.Context().Done(): + timer.Stop() + return nil, req.Context().Err() + case <-timer.C: + } + + if delay < retryMaxDelay { + delay *= 2 + } + } +} + +// shouldRetryRequest reports whether a request should be retried given its result. +func shouldRetryRequest(resp *http.Response, err error) bool { + if err != nil { + // network error, timeout, connection reset, unexpected EOF, etc. + return true + } + switch resp.StatusCode { + case http.StatusInternalServerError, http.StatusBadGateway, + http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + case http.StatusForbidden, http.StatusTooManyRequests: + // Secondary/abuse rate limit signals a Retry-After; retry only then. + // Primary rate limit (no Retry-After) is handled by the downloader. + return resp.Header.Get("Retry-After") != "" + } + return false +} + +func retryReason(resp *http.Response, err error) string { + if err != nil { + return err.Error() + } + return "HTTP " + strconv.Itoa(resp.StatusCode) +} + +// parseRetryAfter parses a Retry-After header expressed in seconds. GitHub uses +// the delta-seconds form; an HTTP-date form (or anything unparsable) yields 0 so +// the caller falls back to exponential backoff. +func parseRetryAfter(v string) time.Duration { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil && secs >= 0 { + return time.Duration(secs) * time.Second + } + return 0 +} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index d4f693f5f4d48..ab51602f93460 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -27901,6 +27901,15 @@ ], "x-go-name": "Service" }, + "sync_issues": { + "description": "only used when mirror is true: keep the mirror's issues/pull requests in\nsync with the remote on each mirror update, shown read-only.", + "type": "boolean", + "x-go-name": "SyncIssues" + }, + "sync_pull_requests": { + "type": "boolean", + "x-go-name": "SyncPullRequests" + }, "uid": { "description": "deprecated (only for backwards compatibility, use repo_owner instead)", "type": "integer", diff --git a/templates/swagger/v1_openapi3_json.tmpl b/templates/swagger/v1_openapi3_json.tmpl index be2f0951093f2..0dafda69d266c 100644 --- a/templates/swagger/v1_openapi3_json.tmpl +++ b/templates/swagger/v1_openapi3_json.tmpl @@ -7605,6 +7605,15 @@ "type": "string", "x-go-name": "Service" }, + "sync_issues": { + "description": "only used when mirror is true: keep the mirror's issues/pull requests in\nsync with the remote on each mirror update, shown read-only.", + "type": "boolean", + "x-go-name": "SyncIssues" + }, + "sync_pull_requests": { + "type": "boolean", + "x-go-name": "SyncPullRequests" + }, "uid": { "deprecated": true, "description": "deprecated (only for backwards compatibility, use repo_owner instead)",