Skip to content

Commit 9d54b14

Browse files
peppypeppy
authored
Merge pull request #12 from bdach/more-data
Add user tag and guest mapper data
2 parents 9b010be + e0995b0 commit 9d54b14

5 files changed

Lines changed: 248 additions & 71 deletions

File tree

‎BeatmapOwnerRow.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
2+
// See the LICENCE file in the repository root for full licence text.
3+
4+
// ReSharper disable InconsistentNaming (intentionally matching database naming)
5+
6+
namespace osu.Server.OnlineDbGenerator
7+
{
8+
public class BeatmapOwnerRow
9+
{
10+
public int user_id { get; set; }
11+
public int beatmap_id { get; set; }
12+
}
13+
}

‎BeatmapTagRow.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
2+
// See the LICENCE file in the repository root for full licence text.
3+
4+
// ReSharper disable InconsistentNaming (intentionally matching database naming)
5+
6+
namespace osu.Server.OnlineDbGenerator
7+
{
8+
public class BeatmapTagRow
9+
{
10+
public int beatmap_id { get; set; }
11+
public int tag_id { get; set; }
12+
}
13+
}

‎Generator.cs‎

Lines changed: 196 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System.Net.Http;
22
using System;
3-
using System.Collections.Generic;
43
using System.Data;
54
using System.IO;
65
using Amazon;
@@ -30,7 +29,9 @@ public class Generator
3029
/// <summary>
3130
/// Conditional to filter beatmaps and beatmapsets by.
3231
/// </summary>
33-
private const string where_conditions = "approved IN (1, 2, 4)";
32+
private const string beatmap_filter_conditions = "approved IN (1, 2, 4)";
33+
34+
private const string beatmap_id_in_filter = $" WHERE `beatmap_id` IN (SELECT `beatmap_id` FROM `osu_beatmaps` WHERE {beatmap_filter_conditions})";
3435

3536
/// <summary>
3637
/// Start generating the online.db file.
@@ -47,6 +48,10 @@ public void Run()
4748

4849
copyBeatmapSets(mysql, sqlite);
4950
copyBeatmaps(mysql, sqlite);
51+
copyTags(mysql, sqlite);
52+
copyBeatmapTags(mysql, sqlite);
53+
copyUsernames(mysql, sqlite);
54+
copyBeatmapOwners(mysql, sqlite);
5055

5156
Console.WriteLine("Compressing...");
5257

@@ -80,136 +85,256 @@ public void Run()
8085
private void createSchema(SqliteConnection sqlite)
8186
{
8287
sqlite.Execute("CREATE TABLE `schema_version` (`number` smallint unsigned NOT NULL)");
83-
sqlite.Execute("INSERT INTO `schema_version` (`number`) VALUES (2)");
84-
85-
sqlite.Execute(@"CREATE TABLE `osu_beatmapsets` (
86-
`beatmapset_id` mediumint unsigned NOT NULL,
87-
`submit_date` timestamp NOT NULL DEFAULT NULL,
88-
`approved_date` timestamp NULL DEFAULT NULL,
89-
`approved` timestamp NULL DEFAULT NULL,
90-
PRIMARY KEY (`beatmapset_id`))");
91-
92-
sqlite.Execute(@"CREATE TABLE `osu_beatmaps` (
93-
`beatmap_id` mediumint unsigned NOT NULL,
94-
`beatmapset_id` mediumint unsigned DEFAULT NULL,
95-
`user_id` int unsigned NOT NULL DEFAULT '0',
96-
`filename` varchar(150) DEFAULT NULL,
97-
`checksum` varchar(32) DEFAULT NULL,
98-
`approved` tinyint NOT NULL DEFAULT '0',
99-
`last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
100-
PRIMARY KEY (`beatmap_id`))");
88+
sqlite.Execute("INSERT INTO `schema_version` (`number`) VALUES (3)");
89+
90+
sqlite.Execute(
91+
"""
92+
CREATE TABLE `osu_beatmapsets` (
93+
`beatmapset_id` mediumint unsigned NOT NULL,
94+
`submit_date` timestamp NOT NULL DEFAULT NULL,
95+
`approved_date` timestamp NULL DEFAULT NULL,
96+
`approved` timestamp NULL DEFAULT NULL,
97+
PRIMARY KEY (`beatmapset_id`))
98+
""");
99+
100+
sqlite.Execute(
101+
"""
102+
CREATE TABLE `osu_beatmaps` (
103+
`beatmap_id` mediumint unsigned NOT NULL,
104+
`beatmapset_id` mediumint unsigned DEFAULT NULL,
105+
`user_id` int unsigned NOT NULL DEFAULT '0',
106+
`filename` varchar(150) DEFAULT NULL,
107+
`checksum` varchar(32) DEFAULT NULL,
108+
`approved` tinyint NOT NULL DEFAULT '0',
109+
`last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
110+
PRIMARY KEY (`beatmap_id`))
111+
""");
101112

102113
sqlite.Execute("CREATE INDEX `beatmapset_id` ON osu_beatmaps (`beatmapset_id`)");
103114
sqlite.Execute("CREATE INDEX `filename` ON osu_beatmaps (`filename`)");
104115
sqlite.Execute("CREATE INDEX `checksum` ON osu_beatmaps (`checksum`)");
105116
sqlite.Execute("CREATE INDEX `user_id` ON osu_beatmaps (`user_id`)");
117+
118+
sqlite.Execute(
119+
"""
120+
CREATE TABLE `tags` (
121+
`id` bigint unsigned NOT NULL,
122+
`name` varchar(255) DEFAULT NULL,
123+
PRIMARY KEY (`id`))
124+
""");
125+
126+
sqlite.Execute(
127+
"""
128+
CREATE TABLE `beatmap_tags` (
129+
`beatmap_id` int unsigned NOT NULL,
130+
`tag_id` int unsigned NOT NULL,
131+
PRIMARY KEY (`beatmap_id`, `tag_id`))
132+
""");
133+
134+
sqlite.Execute(
135+
"""
136+
CREATE TABLE `users` (
137+
`user_id` int unsigned NOT NULL,
138+
`username` varchar(255) DEFAULT NULL,
139+
PRIMARY KEY (`user_id`))
140+
""");
141+
142+
sqlite.Execute(
143+
"""
144+
CREATE TABLE `beatmap_owners` (
145+
`beatmap_id` mediumint unsigned NOT NULL,
146+
`user_id` int unsigned NOT NULL,
147+
PRIMARY KEY (`beatmap_id`, `user_id`))
148+
""");
106149
}
107150

108151
/// <summary>
109152
/// Copy all beatmaps from online MySQL database to cache SQLite database.
110153
/// </summary>
111154
private void copyBeatmapSets(IDbConnection source, IDbConnection destination)
112155
{
113-
int total = getBeatmapSetCount(source);
114-
Console.WriteLine($"Copying {total} beatmap sets...");
156+
int sourceCount = source.QuerySingle<int>($"SELECT COUNT(beatmapset_id) FROM osu_beatmapsets WHERE {beatmap_filter_conditions}", commandTimeout: 600_000);
157+
Console.WriteLine($"Copying {sourceCount} beatmap sets...");
115158

116159
var start = DateTime.Now;
160+
int processedItems = 0;
117161

118162
// only include "permanent" states – ranked, approved, loved.
119163
// this cache may be preferred for initial metadata fetches in lazer so we don't want to include any beatmaps which are still shifting in state.
120-
var beatmapSetsReader = source.Query<BeatmapSetRow>($"SELECT beatmapset_id, approved, approved_date, submit_date FROM osu_beatmapsets WHERE {where_conditions}");
164+
var sourceBeatmapSets = source.Query<BeatmapSetRow>($"SELECT beatmapset_id, approved, approved_date, submit_date FROM osu_beatmapsets WHERE {beatmap_filter_conditions}",
165+
commandTimeout: 600_000);
166+
167+
foreach (var beatmapset in sourceBeatmapSets)
168+
{
169+
destination.Execute("INSERT INTO osu_beatmapsets VALUES(@beatmapset_id, @submit_date, @approved_date, @approved)", beatmapset);
121170

122-
insertBeatmapSets(destination, beatmapSetsReader);
171+
if (++processedItems % 1000 == 0)
172+
Console.WriteLine($"Copied {processedItems} beatmap sets...");
173+
}
123174

124175
var timespan = (DateTime.Now - start).TotalMilliseconds;
176+
int destinationCount = destination.QuerySingle<int>("SELECT COUNT(beatmapset_id) FROM osu_beatmapsets");
125177

126-
int totalSqlite = getBeatmapSetCount(destination);
127-
128-
Console.WriteLine($"Copied beatmap sets in {timespan}ms! (mysql:{total} sqlite:{totalSqlite})");
178+
Console.WriteLine($"Copied beatmap sets in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
129179

130-
if (totalSqlite != total)
131-
{
132-
throw new Exception($"Expected {total} beatmap sets, but found {totalSqlite} in sqlite! Aborting");
133-
}
180+
if (destinationCount != sourceCount)
181+
throw new Exception($"Expected {sourceCount} beatmap sets, but found {destinationCount} in sqlite! Aborting");
134182
}
135183

136184
/// <summary>
137185
/// Copy all beatmaps from online MySQL database to cache SQLite database.
138186
/// </summary>
139187
private void copyBeatmaps(IDbConnection source, IDbConnection destination)
140188
{
141-
int total = getBeatmapCount(source);
142-
Console.WriteLine($"Copying {total} beatmaps...");
189+
int sourceCount = source.QuerySingle<int>($"SELECT COUNT(beatmap_id) FROM osu_beatmaps WHERE {beatmap_filter_conditions}", commandTimeout: 600_000);
190+
Console.WriteLine($"Copying {sourceCount} beatmaps...");
143191

144192
var start = DateTime.Now;
193+
int processedItems = 0;
194+
195+
var sourceBeatmaps = source.Query<BeatmapRow>($"SELECT beatmap_id, beatmapset_id, user_id, filename, checksum, approved, last_update FROM osu_beatmaps WHERE {beatmap_filter_conditions}",
196+
commandTimeout: 600_000);
145197

146-
var beatmapsReader = source.Query<BeatmapRow>($"SELECT beatmap_id, beatmapset_id, user_id, filename, checksum, approved, last_update FROM osu_beatmaps WHERE {where_conditions}");
198+
foreach (var beatmap in sourceBeatmaps)
199+
{
200+
destination.Execute("INSERT INTO osu_beatmaps VALUES(@beatmap_id, @beatmapset_id, @user_id, @filename, @checksum, @approved, @last_update)", beatmap);
147201

148-
insertBeatmaps(destination, beatmapsReader);
202+
if (++processedItems % 1000 == 0)
203+
Console.WriteLine($"Copied {processedItems} beatmaps...");
204+
}
149205

150206
var timespan = (DateTime.Now - start).TotalMilliseconds;
207+
int destinationCount = destination.QuerySingle<int>($"SELECT COUNT(beatmap_id) FROM osu_beatmaps");
151208

152-
int totalSqlite = getBeatmapCount(destination);
209+
Console.WriteLine($"Copied beatmaps in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
153210

154-
Console.WriteLine($"Copied beatmaps in {timespan}ms! (mysql:{total} sqlite:{totalSqlite})");
211+
if (destinationCount != sourceCount)
212+
throw new Exception($"Expected {sourceCount} beatmaps, but found {destinationCount} in sqlite! Aborting");
213+
}
155214

156-
if (totalSqlite != total)
215+
private void copyTags(IDbConnection source, IDbConnection destination)
216+
{
217+
int sourceCount = source.QuerySingle<int>("SELECT COUNT(`id`) FROM `tags`", commandTimeout: 600_000);
218+
Console.WriteLine($"Copying {sourceCount} tags...");
219+
220+
var start = DateTime.Now;
221+
int processedItems = 0;
222+
223+
var sourceTags = source.Query<TagRow>("SELECT `id`, `name` FROM `tags`", commandTimeout: 600_000);
224+
225+
foreach (var tag in sourceTags)
157226
{
158-
throw new Exception($"Expected {total} beatmaps, but found {totalSqlite} in sqlite! Aborting");
227+
destination.Execute("INSERT INTO `tags` VALUES(@id, @name)", tag);
228+
229+
if (++processedItems % 1000 == 0)
230+
Console.WriteLine($"Copied {processedItems} tags...");
159231
}
232+
233+
var timespan = (DateTime.Now - start).TotalMilliseconds;
234+
int destinationCount = destination.QuerySingle<int>("SELECT COUNT(`id`) FROM tags");
235+
236+
Console.WriteLine($"Copied tags in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
237+
238+
if (destinationCount != sourceCount)
239+
throw new Exception($"Expected {sourceCount} tags, but found {destinationCount} in sqlite! Aborting");
160240
}
161241

162-
/// <summary>
163-
/// Insert beatmap sets into the SQLite database.
164-
/// </summary>
165-
/// <param name="conn">Connection to insert beatmaps into.</param>
166-
/// <param name="beatmaps">DbDataReader object (obtained from SelectBeatmaps) to insert beatmaps from.</param>
167-
private void insertBeatmapSets(IDbConnection conn, IEnumerable<BeatmapSetRow> beatmapsets)
242+
private void copyBeatmapTags(IDbConnection source, IDbConnection destination)
168243
{
169-
const string sql = "INSERT INTO osu_beatmapsets VALUES(@beatmapset_id, @submit_date, @approved_date, @approved)";
244+
int sourceCount = source.QuerySingle<int>(
245+
$"""
246+
SELECT COUNT(DISTINCT `beatmap_id`, `tag_id`) FROM `beatmap_tags`
247+
{beatmap_id_in_filter}
248+
""", commandTimeout: 600_000);
249+
Console.WriteLine($"Copying {sourceCount} beatmap tag pairs...");
170250

251+
var start = DateTime.Now;
171252
int processedItems = 0;
172253

173-
foreach (var beatmapset in beatmapsets)
254+
var sourceBeatmapTags = source.Query<BeatmapTagRow>(
255+
$"""
256+
SELECT DISTINCT `beatmap_id`, `tag_id` FROM `beatmap_tags`
257+
{beatmap_id_in_filter}
258+
""", commandTimeout: 600_000);
259+
260+
foreach (var beatmapTag in sourceBeatmapTags)
174261
{
175-
conn.Execute(sql, beatmapset);
262+
destination.Execute("INSERT INTO `beatmap_tags` VALUES(@beatmap_id, @tag_id)", beatmapTag);
176263

177-
if (++processedItems % 50 == 0)
178-
Console.WriteLine($"Copied {processedItems} beatmap sets...");
264+
if (++processedItems % 1000 == 0)
265+
Console.WriteLine($"Copied {processedItems} tags...");
179266
}
267+
268+
var timespan = (DateTime.Now - start).TotalMilliseconds;
269+
int destinationCount = destination.QuerySingle<int>("SELECT COUNT(1) FROM `beatmap_tags`");
270+
271+
Console.WriteLine($"Copied beatmap tags in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
272+
273+
if (destinationCount != sourceCount)
274+
throw new Exception($"Expected {sourceCount} beatmap tags, but found {destinationCount} in sqlite! Aborting");
180275
}
181276

182-
/// <summary>
183-
/// Insert beatmaps into the SQLite database.
184-
/// </summary>
185-
/// <param name="conn">Connection to insert beatmaps into.</param>
186-
/// <param name="beatmaps">DbDataReader object (obtained from SelectBeatmaps) to insert beatmaps from.</param>
187-
private void insertBeatmaps(IDbConnection conn, IEnumerable<BeatmapRow> beatmaps)
277+
private void copyUsernames(IDbConnection source, IDbConnection destination)
188278
{
189-
const string sql = "INSERT INTO osu_beatmaps VALUES(@beatmap_id, @beatmapset_id, @user_id, @filename, @checksum, @approved, @last_update)";
279+
int sourceCount = source.QuerySingle<int>(
280+
$"""
281+
SELECT COUNT(`user_id`) FROM `phpbb_users`
282+
WHERE `user_id` IN (SELECT `user_id` FROM `osu_beatmaps` WHERE {beatmap_filter_conditions} UNION SELECT `user_id` FROM `beatmap_owners` {beatmap_id_in_filter})
283+
""", commandTimeout: 600_000);
284+
Console.WriteLine($"Copying {sourceCount} usernames...");
190285

286+
var start = DateTime.Now;
191287
int processedItems = 0;
192288

193-
foreach (var beatmap in beatmaps)
289+
var sourceUsers = source.Query<UserRow>(
290+
$"""
291+
SELECT `user_id`, `username` FROM `phpbb_users`
292+
WHERE `user_id` IN (SELECT `user_id` FROM `osu_beatmaps` WHERE {beatmap_filter_conditions} UNION SELECT `user_id` FROM `beatmap_owners` {beatmap_id_in_filter})
293+
""", commandTimeout: 600_000);
294+
295+
foreach (var user in sourceUsers)
194296
{
195-
conn.Execute(sql, beatmap);
297+
destination.Execute("INSERT INTO `users` VALUES(@user_id, @username)", user);
196298

197-
if (++processedItems % 50 == 0)
198-
Console.WriteLine($"Copied {processedItems} beatmaps...");
299+
if (++processedItems % 1000 == 0)
300+
Console.WriteLine($"Copied {processedItems} usernames...");
199301
}
302+
303+
var timespan = (DateTime.Now - start).TotalMilliseconds;
304+
int destinationCount = destination.QuerySingle<int>("SELECT COUNT(`user_id`) FROM `users`");
305+
306+
Console.WriteLine($"Copied usernames in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
307+
308+
if (destinationCount != sourceCount)
309+
throw new Exception($"Expected {sourceCount} usernames, but found {destinationCount} in sqlite! Aborting");
200310
}
201311

202-
/// <summary>
203-
/// Count beatmap sets from MySQL or SQLite database.
204-
/// </summary>
205-
/// <param name="conn">Connection to fetch beatmaps from.</param>
206-
private int getBeatmapSetCount(IDbConnection conn) => conn.QuerySingle<int>($"SELECT COUNT(beatmapset_id) FROM osu_beatmapsets WHERE {where_conditions}");
312+
private void copyBeatmapOwners(IDbConnection source, IDbConnection destination)
313+
{
314+
int sourceCount = source.QuerySingle<int>($"SELECT COUNT(1) FROM `beatmap_owners` {beatmap_id_in_filter}", commandTimeout: 600_000);
315+
Console.WriteLine($"Copying {sourceCount} beatmap owners...");
207316

208-
/// <summary>
209-
/// Count beatmaps from MySQL or SQLite database.
210-
/// </summary>
211-
/// <param name="conn">Connection to fetch beatmaps from.</param>
212-
private int getBeatmapCount(IDbConnection conn) => conn.QuerySingle<int>($"SELECT COUNT(beatmap_id) FROM osu_beatmaps WHERE {where_conditions}");
317+
var start = DateTime.Now;
318+
int processedItems = 0;
319+
320+
var sourceBeatmapOwners = source.Query<BeatmapOwnerRow>($"SELECT `beatmap_id`, `user_id` FROM `beatmap_owners` {beatmap_id_in_filter}", commandTimeout: 600_000);
321+
322+
foreach (var owner in sourceBeatmapOwners)
323+
{
324+
destination.Execute("INSERT INTO `beatmap_owners` VALUES(@beatmap_id, @user_id)", owner);
325+
326+
if (++processedItems % 1000 == 0)
327+
Console.WriteLine($"Copied {processedItems} beatmap owners...");
328+
}
329+
330+
var timespan = (DateTime.Now - start).TotalMilliseconds;
331+
int destinationCount = destination.QuerySingle<int>("SELECT COUNT(1) FROM `beatmap_owners`");
332+
333+
Console.WriteLine($"Copied beatmap owners in {timespan}ms! (mysql:{sourceCount} sqlite:{destinationCount})");
334+
335+
if (destinationCount != sourceCount)
336+
throw new Exception($"Expected {sourceCount} beatmap owners, but found {destinationCount} in sqlite! Aborting");
337+
}
213338

214339
/// <summary>
215340
/// Get a connection to the offline SQLite cache database.

‎TagRow.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
2+
// See the LICENCE file in the repository root for full licence text.
3+
4+
// ReSharper disable InconsistentNaming (intentionally matching database naming)
5+
6+
namespace osu.Server.OnlineDbGenerator
7+
{
8+
public class TagRow
9+
{
10+
public long id { get; set; }
11+
public string name { get; set; }
12+
}
13+
}

0 commit comments

Comments
 (0)