From 02a7a33d2697985493f93ba7c33ef9c3a3ceb473 Mon Sep 17 00:00:00 2001 From: Joshua Gerrish Date: Thu, 10 Feb 2022 23:10:26 +0000 Subject: [PATCH 1/2] Fix issue 367 https://github.com/sharkwouter/minigalaxy/issues/367 This adds code to cache cover images to disk It also adds new AssetManager and Asset classes to manage assets like thumbnails and covers. Tests have been added for Asset, some additional mocking needs to be added for network tests. --- minigalaxy/asset_manager.py | 220 +++++++++++++++++++++++++++++++++ minigalaxy/download_manager.py | 1 + minigalaxy/paths.py | 1 + minigalaxy/ui/information.py | 30 +++-- minigalaxy/ui/window.py | 8 +- tests/test_asset_manager.py | 205 ++++++++++++++++++++++++++++++ 6 files changed, 445 insertions(+), 20 deletions(-) create mode 100644 minigalaxy/asset_manager.py create mode 100644 tests/test_asset_manager.py diff --git a/minigalaxy/asset_manager.py b/minigalaxy/asset_manager.py new file mode 100644 index 00000000..42b1f5fd --- /dev/null +++ b/minigalaxy/asset_manager.py @@ -0,0 +1,220 @@ +from datetime import datetime, timedelta +from enum import Enum +import os +from pathlib import PurePath +import urllib +from minigalaxy.download import Download +from minigalaxy.download_manager import DownloadManager +from minigalaxy.paths import COVER_DIR, THUMBNAIL_DIR +from minigalaxy.ui.gtk import Gio, GdkPixbuf + + +class ItemNotCachedError(Exception): + """Raised when an item is not found or not cached""" + pass + + +class AssetType(Enum): + ICON = 1 + THUMBNAIL = 2 + COVER = 3 + + +class Asset: + """ + Class for a single asset such as a thumbnail + This class provides methods for checking if an asset already exists in the filesystem + and whether it's expired. + """ + def __init__(self, asset_type, url, extra={}): + """ + Create a new asset + The constructor accepts two required arguments and an optional argument + The first argument is the AssetType, such as AssetType.THUMBNAIL + The second argument is the URL of the asset to download + The third argument is optional data that can be used to construct + the filename or alternate sources. + """ + self.asset_type = asset_type + self.url = url + self.extra = extra + self.build_filename() + + def url_file_extension(self): + """ + Get the file extension from the URL path extension + """ + if self.url: + url_path = urllib.parse.urlparse(self.url).path + file_path = PurePath(url_path) + extension = file_path.suffix.lstrip(".") + return extension + else: + return None + + def build_filename(self): + """ + Build the filename from the AssetType and image URL + """ + if self.asset_type == AssetType.COVER: + if "game_id" in self.extra: + extension = self.url_file_extension() + if extension: + self.filename = os.path.join(COVER_DIR, "{}.{}".format( + self.extra["game_id"], extension)) + else: + self.filename = None + if "game_installed" in self.extra and self.extra["game_installed"] and "game_install_dir" in self.extra: + self.alt_filename = os.path.join(self.extra["game_install_dir"], "cover.jpg") + + def exists(self): + "Return True if the file exists" + if self.filename: + return os.path.exists(self.filename) + else: + return False + + def modified_time(self): + "Return the last modified time (mtime) of the asset" + try: + t = datetime.fromtimestamp(os.stat(self.filename).st_mtime) + return t + except FileNotFoundError: + raise ItemNotCachedError(self.filename) + + def expired(self): + "Return True if an item is expired, otherwise return False" + elapsed = datetime.now() - self.modified_time() + return elapsed > timedelta(days=1) + + +class AssetManager: + """ + Manage a set of assets such as icons, thumbnails, games and DLC + This class handles downloading, caching, resizing, saving and loading assets. + + This class has one main method that users should call after creating the object: + + Example: + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + { "game_id": self.api_info[0]["id"], + "game_installed": False + }) + am = AssetManager(asset) + am.load(draw_callback) + + Where draw_callback is a function or method that accepts a Gtk pixbuf + """ + def __init__(self, asset): + """ + Construct the AssetManager with a single Asset + """ + self.asset = asset + + def create_asset_dirs(): + """ + Class method to create the asset directories + + Example: + AssetManager.create_asset_dirs() + """ + # Create the thumbnails directory + if not os.path.exists(THUMBNAIL_DIR): + os.makedirs(THUMBNAIL_DIR, mode=0o755) + + # Create the covers directory + if not os.path.exists(COVER_DIR): + os.makedirs(COVER_DIR, mode=0o755) + + def load(self, draw_callback): + """ + Load an asset and call a callback to draw the pixbuf + If the image isn't cached, download it, resize it, save it and display it + If the image is cached, load the file and display it + + Users of this class should pass in a function that accepts a Gtk pixbuf argument + The class will pass in the asset after loading it. + """ + self.__draw = draw_callback + + # Using a try except pattern for cache misses simplifies the logic a bit + try: + if self.asset.exists(): + if self.asset.expired(): + raise ItemNotCachedError + else: + self.__load_asset() + else: + # Case where the asset doesn't exist on the filesystem + raise ItemNotCachedError + except ItemNotCachedError: + if self.asset.url: + self.download_asset() + else: + # If there is no URL, we can try using the installed game + # thumbnail + self.__load_alternate_asset() + + def download_thumbnail(self): + response = urllib.request.urlopen(self.asset.url) + input_stream = Gio.MemoryInputStream.new_from_data(response.read(), None) + pixbuf = GdkPixbuf.Pixbuf.new_from_stream(input_stream, None) + pixbuf = pixbuf.scale_simple(340, 480, GdkPixbuf.InterpType.BILINEAR) + + def download_asset(self): + """ + Download an asset image + After downloading, the asset is resized, saved and drawn to the window + """ + download = Download(self.asset.url, self.asset.filename, + finish_func=self.__resize_asset) + DownloadManager.download_now(download) + + def __resize_asset(self, save_location): + """ + Resize a asset image + After resizing, the asset is saved and drawn to the window + """ + if not os.path.isfile(self.asset.filename): + return + pixbuf = GdkPixbuf.Pixbuf.new_from_file(self.asset.filename) + pixbuf = pixbuf.scale_simple(340, 480, GdkPixbuf.InterpType.BILINEAR) + self.__save_asset(pixbuf) + + def __save_asset(self, pixbuf): + """ + Save an asset image + After saving, the asset is drawn to the window + """ + extension = self.asset.url_file_extension() + pixbuf.savev(self.asset.filename, extension) + self.__draw(pixbuf) + + def __load_asset(self): + """ + Load an asset image + After getting the filename, the asset loaded and drawn to the window + """ + asset_path = self.asset.filename + if not os.path.isfile(asset_path): + self.__load_alternate_asset() + else: + self.__load_file(asset_path) + + def __load_alternate_asset(self): + """ + Load an alternate asset if one is available + """ + if self.asset.alt_filename: + asset_path = self.asset.alt_filename + if not os.path.isfile(asset_path): + return + self.__load_file(asset_path) + + def __load_file(self, asset_path): + "Finally load the asset from the filesystem and call the callback with the pixbuf" + try: + pixbuf = GdkPixbuf.Pixbuf.new_from_file(asset_path) + self.__draw(pixbuf) + except FileNotFoundError: + return diff --git a/minigalaxy/download_manager.py b/minigalaxy/download_manager.py index 3f663ad4..2b33bd97 100644 --- a/minigalaxy/download_manager.py +++ b/minigalaxy/download_manager.py @@ -131,6 +131,7 @@ def download_operation(self, download, start_point, download_mode): resume_header = {'Range': 'bytes={}-'.format(start_point)} download_request = SESSION.get(download.url, headers=resume_header, stream=True, timeout=30) downloaded_size = start_point + # TODO: The content-length header may be missing (None returned) file_size = int(download_request.headers.get('content-length')) result = True if downloaded_size < file_size: diff --git a/minigalaxy/paths.py b/minigalaxy/paths.py index fd8bdfe8..864b0d35 100644 --- a/minigalaxy/paths.py +++ b/minigalaxy/paths.py @@ -11,6 +11,7 @@ CACHE_DIR = os.path.join(os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')), "minigalaxy") THUMBNAIL_DIR = os.path.join(CACHE_DIR, "thumbnails") +COVER_DIR = os.path.join(CACHE_DIR, "covers") APPLICATIONS_DIR = os.path.expanduser("~/.local/share/applications") DEFAULT_INSTALL_DIR = os.path.expanduser("~/GOG Games") diff --git a/minigalaxy/ui/information.py b/minigalaxy/ui/information.py index ab5184d3..a10e41ef 100644 --- a/minigalaxy/ui/information.py +++ b/minigalaxy/ui/information.py @@ -1,11 +1,11 @@ -import urllib import os import webbrowser -from minigalaxy.paths import UI_DIR, THUMBNAIL_DIR +from minigalaxy.paths import UI_DIR from minigalaxy.translation import _ from minigalaxy.config import Config -from minigalaxy.ui.gtk import Gtk, GLib, Gio, GdkPixbuf +from minigalaxy.ui.gtk import Gtk, GLib +from minigalaxy.asset_manager import Asset, AssetType, AssetManager @Gtk.Template.from_file(os.path.join(UI_DIR, "information.ui")) @@ -31,7 +31,7 @@ def __init__(self, parent, game, api): self.gamesdb_info = self.api.get_gamesdb_info(self.game) # Show the image - self.load_thumbnail() + self.load_cover() self.load_description() # Center information window @@ -91,18 +91,16 @@ def on_menu_button_pcgamingwiki(self, widget): _("Please check your internet connection") ) - def load_thumbnail(self): - if self.gamesdb_info["cover"]: - response = urllib.request.urlopen(self.gamesdb_info["cover"]) - input_stream = Gio.MemoryInputStream.new_from_data(response.read(), None) - pixbuf = GdkPixbuf.Pixbuf.new_from_stream(input_stream, None) - pixbuf = pixbuf.scale_simple(340, 480, GdkPixbuf.InterpType.BILINEAR) - GLib.idle_add(self.image.set_from_pixbuf, pixbuf) - else: - thumbnail_path = os.path.join(THUMBNAIL_DIR, "{}.jpg".format(self.game.id)) - if not os.path.isfile(thumbnail_path) and self.game.is_installed: - thumbnail_path = os.path.join(self.game.install_dir, "thumbnail.jpg") - GLib.idle_add(self.image.set_from_file, thumbnail_path) + def load_cover(self): + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.game.id, + "game_installed": self.game.is_installed(), + "game_install_dir": self.game.install_dir}) + asset_manager = AssetManager(asset) + asset_manager.load(self.draw_cover) + + def draw_cover(self, pixbuf): + GLib.idle_add(self.image.set_from_pixbuf, pixbuf) def load_description(self): description = "" diff --git a/minigalaxy/ui/window.py b/minigalaxy/ui/window.py index 893cb7cf..65815ca7 100644 --- a/minigalaxy/ui/window.py +++ b/minigalaxy/ui/window.py @@ -1,12 +1,13 @@ import os import locale +from minigalaxy.asset_manager import AssetManager from minigalaxy.ui.login import Login from minigalaxy.ui.preferences import Preferences from minigalaxy.ui.about import About from minigalaxy.api import Api from minigalaxy.config import Config -from minigalaxy.paths import UI_DIR, LOGO_IMAGE_PATH, THUMBNAIL_DIR +from minigalaxy.paths import UI_DIR, LOGO_IMAGE_PATH from minigalaxy.translation import _ from minigalaxy.ui.library import Library from minigalaxy.ui.gtk import Gtk, Gdk, GdkPixbuf @@ -62,9 +63,8 @@ def __init__(self, name="Minigalaxy"): self.maximize() self.show_all() - # Create the thumbnails directory - if not os.path.exists(THUMBNAIL_DIR): - os.makedirs(THUMBNAIL_DIR, mode=0o755) + # Create the cover and thumbnail directories and any other needed ones + AssetManager.create_asset_dirs() # Interact with the API self.offline = not self.api.can_connect() diff --git a/tests/test_asset_manager.py b/tests/test_asset_manager.py new file mode 100644 index 00000000..ad1eacdf --- /dev/null +++ b/tests/test_asset_manager.py @@ -0,0 +1,205 @@ +""" +Test the asset_manager module + +Similar to the test_api.py test suite, we mock out the API calls +In addition, we mock filesystem calls and Gtk calls. +""" +from datetime import datetime, timedelta +import math +import os +import sys +import time +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import MagicMock, patch + +m_constants = MagicMock() +m_config = MagicMock() +m_paths = MagicMock() +m_gtk = MagicMock() +m_gi = MagicMock() +m_gametile = MagicMock() +m_library = MagicMock() +m_preferences = MagicMock() +m_login = MagicMock() +m_about = MagicMock() +m_window = MagicMock() + +sys.modules['minigalaxy.constants'] = m_constants +sys.modules['minigalaxy.config'] = m_config +sys.modules['minigalaxy.paths'] = m_paths +sys.modules['minigalaxy.ui.window'] = m_window +sys.modules['minigalaxy.ui.preferences'] = m_preferences +sys.modules['minigalaxy.ui.gametile'] = m_gametile + + +class UnitTestGtkTemplate: + + def __init__(self): + self.Child = m_gtk + + def from_file(self, lib_file): + def passthrough(func): + def passthrough2(): + return func() + return passthrough2 + return passthrough + + Callback = MagicMock() + + +class UnitTestGiRepository: + + class Gtk: + Template = UnitTestGtkTemplate() + Widget = MagicMock() + Settings = MagicMock() + ResponseType = MagicMock() + + class ApplicationWindow: + def __init__(self, title): + pass + + set_default_icon_list = MagicMock() + show_all = MagicMock() + + Gdk = MagicMock() + GdkPixbuf = MagicMock() + Gio = MagicMock() + GLib = MagicMock + + +u_gi_repository = UnitTestGiRepository() +sys.modules['gi.repository'] = u_gi_repository +sys.modules['gi'] = m_gi +sys.modules['minigalaxy.ui.library'] = m_library +sys.modules['minigalaxy.ui.preferences'] = m_preferences +sys.modules['minigalaxy.ui.login'] = m_login +sys.modules['minigalaxy.ui.about'] = m_about +sys.modules['minigalaxy.ui.gtk'] = u_gi_repository + +from minigalaxy.api import Api # noqa: E402 +from minigalaxy.game import Game # noqa: E402 +from minigalaxy.paths import COVER_DIR # noqa: E402 +from minigalaxy.asset_manager import Asset, AssetType # noqa: E402 + +API_GET_INFO_STELLARIS = [{'id': '51622789000874509', 'game_id': '51154268886064420', 'platform_id': 'gog', 'external_id': '1508702879', 'game': {'genres': [{'id': '51071904337940794', 'name': {'*': 'Strategy', 'en-US': 'Strategy'}, 'slug': 'strategy'}], 'summary': {'*': 'Stellaris description'}, 'visible_in_library': True, 'aggregated_rating': 78.5455, 'game_modes': [{'id': '53051895165351137', 'name': 'Single player', 'slug': 'single-player'}, {'id': '53051908711988230', 'name': 'Multiplayer', 'slug': 'multiplayer'}], 'horizontal_artwork': {'url_format': 'https://images.gog.com/742acfb77ec51ca48c9f96947bf1fc0ad8f0551c9c9f338021e8baa4f08e449f{formatter}.{ext}?namespace=gamesdb'}, 'background': {'url_format': 'https://images.gog.com/742acfb77ec51ca48c9f96947bf1fc0ad8f0551c9c9f338021e8baa4f08e449f{formatter}.{ext}?namespace=gamesdb'}, 'vertical_cover': {'url_format': 'https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2{formatter}.{ext}?namespace=gamesdb'}, 'cover': {'url_format': 'https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2{formatter}.{ext}?namespace=gamesdb'}, 'logo': {'url_format': 'https://images.gog.com/c50a5d26c42d84b4b884976fb89d10bb3e97ebda0c0450285d92b8c50844d788{formatter}.{ext}?namespace=gamesdb'}, 'icon': {'url_format': 'https://images.gog.com/c85cf82e6019dd52fcdf1c81d17687dd52807835f16aa938abd2a34e5d9b99d0{formatter}.{ext}?namespace=gamesdb'}, 'square_icon': {'url_format': 'https://images.gog.com/c3adc81bf37f1dd89c9da74c13967a08b9fd031af4331750dbc65ab0243493c8{formatter}.{ext}?namespace=gamesdb'}}}] +GAMESDB_INFO_STELLARIS = {'cover': 'https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb', 'vertical_cover': 'https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb', 'background': 'https://images.gog.com/742acfb77ec51ca48c9f96947bf1fc0ad8f0551c9c9f338021e8baa4f08e449f.png?namespace=gamesdb', 'summary': {'*': 'Stellaris description'}, 'genre': {'*': 'Strategy', 'en-US': 'Strategy'}} + + +class TestAsset(TestCase): + """ + Test the AssetManager classes + """ + def setUp(self): + """ + Use the same API request mock for most of the tests + """ + api = Api() + api._Api__request_gamesdb = MagicMock() + api._Api__request_gamesdb.side_effect = API_GET_INFO_STELLARIS + api.get_info = MagicMock() + api.get_info.return_value = API_GET_INFO_STELLARIS + test_game = Game("Stellaris") + self.api_info = api.get_info(test_game) + self.gamesdb_info = api.get_gamesdb_info(test_game) + + def test_asset_parses_gamedb(self): + """ + Test that the Asset class creates and Asset and generates correct + filenames from URLs. + """ + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.api_info[0]["id"], + "game_installed": False}) + + self.assertEqual(asset.url, "https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb") + self.assertEqual(asset.url_file_extension(), "png") + self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) + + @patch('os.path') + def test_asset_file_exists(self, path_mock): + """ + Test that the Asset class correctly checks existence for files + """ + fn = os.path.join(COVER_DIR, "51622789000874509.png") + path_mock.exists.return_value = True + + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.api_info[0]["id"], + "game_installed": False}) + + self.assertEqual(asset.url, "https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb") + self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) + self.assertEqual(asset.exists(), True) + path_mock.exists.assert_called_with(fn) + + @patch('os.path') + def test_asset_file_doesnt_exist(self, path_mock): + """ + Test that the Asset class correctly checks for existence of files + """ + fn = os.path.join(COVER_DIR, "51622789000874509.png") + path_mock.exists.return_value = False + + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.api_info[0]["id"], + "game_installed": False}) + + self.assertEqual(asset.url, "https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb") + self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) + self.assertEqual(asset.exists(), False) + path_mock.exists.assert_called_with(fn) + + @patch('os.path') + @patch('os.stat') + def test_asset_not_expired(self, stat_mock, path_mock): + """ + Test that the Asset class correctly checks that files are not expired + """ + path_mock.exists.return_value = True + + # Using datetime.now() for testing, this makes some assumptions like our test + # doesn't run longer than the cache expiration time we set + stat_result = SimpleNamespace(st_mode=33188, st_ino=7876932, st_dev=234881026, + st_nlink=1, st_uid=501, st_gid=501, st_size=264, + st_atime=1297230295, st_mtime=math.floor(time.time()), + st_ctime=1297230027) + + stat_mock.return_value = stat_result + + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.api_info[0]["id"], + "game_installed": False}) + + self.assertEqual(asset.url, "https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb") + self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) + self.assertEqual(asset.exists(), True) + self.assertEqual(asset.expired(), False) + + @patch('os.path') + @patch('os.stat') + def test_asset_expired(self, stat_mock, path_mock): + """ + Test that the Asset class correctly checks that files are expired + """ + path_mock.exists.return_value = True + + # Using datetime.now() for testing, this makes some assumptions like our test + # doesn't run longer than the cache expiration time we set + stat_result = SimpleNamespace(st_mode=33188, st_ino=7876932, st_dev=234881026, + st_nlink=1, st_uid=501, st_gid=501, st_size=264, + st_atime=1297230295, + st_mtime=math.floor((datetime.now() - timedelta(days=2)).timestamp()), + st_ctime=1297230027) + + stat_mock.return_value = stat_result + + asset = Asset(AssetType.COVER, self.gamesdb_info["cover"], + {"game_id": self.api_info[0]["id"], + "game_installed": False}) + + self.assertEqual(asset.url, "https://images.gog.com/8d822a05746670fb2540e9c136f0efaed6a2d5ab698a9f8bd7f899d21f2022d2.png?namespace=gamesdb") + self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) + self.assertEqual(asset.exists(), True) + self.assertEqual(asset.expired(), True) From 0fa12f5526949b8254403cf9a370d6bca9f8c037 Mon Sep 17 00:00:00 2001 From: Joshua Gerrish Date: Mon, 14 Feb 2022 20:54:54 +0000 Subject: [PATCH 2/2] Clean up mocks so tests run successfully. --- tests/test_asset_manager.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_asset_manager.py b/tests/test_asset_manager.py index ad1eacdf..9e9e2c0e 100644 --- a/tests/test_asset_manager.py +++ b/tests/test_asset_manager.py @@ -203,3 +203,21 @@ def test_asset_expired(self, stat_mock, path_mock): self.assertEqual(asset.filename, os.path.join(COVER_DIR, "51622789000874509.png")) self.assertEqual(asset.exists(), True) self.assertEqual(asset.expired(), True) + + +# These mocks are polluting the test namespace when tests are run +# with python -m unittest tests/*.py +# So make sure we clean them up +del sys.modules['gi'] +del sys.modules['gi.repository'] +del sys.modules['minigalaxy.constants'] +del sys.modules['minigalaxy.config'] +del sys.modules['minigalaxy.paths'] +del sys.modules['minigalaxy.ui.gametile'] +del sys.modules['minigalaxy.ui.library'] +del sys.modules['minigalaxy.ui.preferences'] +del sys.modules['minigalaxy.ui.login'] +del sys.modules['minigalaxy.ui.about'] +del sys.modules['minigalaxy.ui.gtk'] +del sys.modules['minigalaxy.ui.window'] +del sys.modules['minigalaxy.game']