feat(assets): Delete original assets unused outside of the optimization pipeline (withastro#8954)

Co-authored-by: Sarah Rainsberger <[email protected]>

Author: Princesseuh

.changeset/nasty-elephants-provide.md

@@ -0,0 +1,5 @@
+---
+'astro': minor
+---
+
+Updates the Image Services API to now delete original images from the final build that are not used outside of the optimization pipeline. For users with a large number of these images (e.g. thumbnails), this should reduce storage consumption and deployment times.

packages/astro/src/assets/build/generate.ts

@@ -31,6 +31,7 @@ type GenerationData = GenerationDataUncached | GenerationDataCached;
 
 type AssetEnv = {
 	logger: Logger;
+	isSSR: boolean;
 	count: { total: number; current: number };
 	useCache: boolean;
 	assetsCacheDir: URL;
@@ -74,6 +75,7 @@ export async function prepareAssetsGenerationEnv(
 
 	return {
 		logger,
+		isSSR: isServerLikeOutput(config),
 		count,
 		useCache,
 		assetsCacheDir,
@@ -84,20 +86,41 @@ export async function prepareAssetsGenerationEnv(
 	};
 }
 
+function getFullImagePath(originalFilePath: string, env: AssetEnv): URL {
+	return new URL(
+		'.' + prependForwardSlash(join(env.assetsFolder, basename(originalFilePath))),
+		env.serverRoot
+	);
+}
+
 export async function generateImagesForPath(
 	originalFilePath: string,
-	transforms: MapValue<AssetsGlobalStaticImagesList>,
+	transformsAndPath: MapValue<AssetsGlobalStaticImagesList>,
 	env: AssetEnv,
 	queue: PQueue
 ) {
 	const originalImageData = await loadImage(originalFilePath, env);
 
-	for (const [_, transform] of transforms) {
+	for (const [_, transform] of transformsAndPath.transforms) {
 		queue.add(async () =>
 			generateImage(originalImageData, transform.finalPath, transform.transform)
 		);
 	}
 
+	// In SSR, we cannot know if an image is referenced in a server-rendered page, so we can't delete anything
+	// For instance, the same image could be referenced in both a server-rendered page and build-time-rendered page
+	if (
+		!env.isSSR &&
+		!isRemotePath(originalFilePath) &&
+		!globalThis.astroAsset.referencedImages?.has(transformsAndPath.originalSrcPath)
+	) {
+		try {
+			await fs.promises.unlink(getFullImagePath(originalFilePath, env));
+		} catch (e) {
+			/* No-op, it's okay if we fail to delete one of the file, we're not too picky. */
+		}
+	}
+
 	async function generateImage(
 		originalImage: ImageData,
 		filepath: string,
@@ -245,9 +268,7 @@ async function loadImage(path: string, env: AssetEnv): Promise<ImageData> {
 	}
 
 	return {
-		data: await fs.promises.readFile(
-			new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot)
-		),
+		data: await fs.promises.readFile(getFullImagePath(path, env)),
 		expires: 0,
 	};
 }

packages/astro/src/assets/internal.ts

@@ -91,6 +91,15 @@ export async function getImage(
 				: options.src,
 	};
 
+	// Clone the `src` object if it's an ESM import so that we don't refer to any properties of the original object
+	// Causing our generate step to think the image is used outside of the image optimization pipeline
+	const clonedSrc = isESMImportedImage(resolvedOptions.src)
+		? // @ts-expect-error - clone is a private, hidden prop
+		  resolvedOptions.src.clone ?? resolvedOptions.src
+		: resolvedOptions.src;
+
+	resolvedOptions.src = clonedSrc;
+
 	const validatedOptions = service.validateOptions
 		? await service.validateOptions(resolvedOptions, imageConfig)
 		: resolvedOptions;

packages/astro/src/assets/types.ts

@@ -10,7 +10,10 @@ export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string
 
 export type AssetsGlobalStaticImagesList = Map<
 	string,
-	Map<string, { finalPath: string; transform: ImageTransform }>
+	{
+		originalSrcPath: string;
+		transforms: Map<string, { finalPath: string; transform: ImageTransform }>;
+	}
 >;
 
 declare global {
@@ -19,6 +22,7 @@ declare global {
 		imageService?: ImageService;
 		addStaticImage?: ((options: ImageTransform, hashProperties: string[]) => string) | undefined;
 		staticImages?: AssetsGlobalStaticImagesList;
+		referencedImages?: Set<string>;
 	};
 }
 
@@ -31,6 +35,8 @@ export interface ImageMetadata {
 	height: number;
 	format: ImageInputFormat;
 	orientation?: number;
+	/** @internal */
+	fsPath: string;
 }
 
 /**

packages/astro/src/assets/utils/emitAsset.ts

@@ -24,11 +24,18 @@ export async function emitESMImage(
 
 	const fileMetadata = await imageMetadata(fileData, id);
 
-	const emittedImage: ImageMetadata = {
+	const emittedImage: Omit<ImageMetadata, 'fsPath'> = {
 		src: '',
 		...fileMetadata,
 	};
 
+	// Private for now, we generally don't want users to rely on filesystem paths, but we need it so that we can maybe remove the original asset from the build if it's unused.
+	Object.defineProperty(emittedImage, 'fsPath', {
+		enumerable: false,
+		writable: false,
+		value: url,
+	});
+
 	// Build
 	if (!watchMode) {
 		const pathname = decodeURI(url.pathname);
@@ -50,7 +57,7 @@ export async function emitESMImage(
 		emittedImage.src = `/@fs` + prependForwardSlash(fileURLToNormalizedPath(url));
 	}
 
-	return emittedImage;
+	return emittedImage as ImageMetadata;
 }
 
 function fileURLToNormalizedPath(filePath: URL): string {

packages/astro/src/assets/utils/metadata.ts

@@ -5,7 +5,7 @@ import type { ImageInputFormat, ImageMetadata } from '../types.js';
 export async function imageMetadata(
 	data: Buffer,
 	src?: string
-): Promise<Omit<ImageMetadata, 'src'>> {
+): Promise<Omit<ImageMetadata, 'src' | 'fsPath'>> {
 	const result = probe.sync(data);
 
 	if (result === null) {

packages/astro/src/assets/utils/proxy.ts

@@ -0,0 +1,13 @@
+export function getProxyCode(options: Record<string, any>, isSSR: boolean): string {
+	return `
+						new Proxy(${JSON.stringify(options)}, {
+						get(target, name, receiver) {
+							if (name === 'clone') {
+								return structuredClone(target);
+							}
+							${!isSSR ? 'globalThis.astroAsset.referencedImages.add(target.fsPath);' : ''}
+							return target[name];
+						}
+					})
+					`;
+}

packages/astro/src/assets/utils/queryParams.ts

@@ -2,7 +2,7 @@ import type { ImageInputFormat, ImageMetadata } from '../types.js';
 
 export function getOrigQueryParams(
 	params: URLSearchParams
-): Omit<ImageMetadata, 'src'> | undefined {
+): Pick<ImageMetadata, 'width' | 'height' | 'format'> | undefined {
 	const width = params.get('origWidth');
 	const height = params.get('origHeight');
 	const format = params.get('origFormat');

packages/astro/src/assets/vite-plugin-assets.ts

@@ -14,6 +14,7 @@ import { isServerLikeOutput } from '../prerender/utils.js';
 import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
 import { isESMImportedImage } from './internal.js';
 import { emitESMImage } from './utils/emitAsset.js';
+import { getProxyCode } from './utils/proxy.js';
 import { hashTransform, propsToFilename } from './utils/transformToPath.js';
 
 const resolvedVirtualModuleId = '\0' + VIRTUAL_MODULE_ID;
@@ -26,7 +27,9 @@ export default function assets({
 }: AstroPluginOptions & { mode: string }): vite.Plugin[] {
 	let resolvedConfig: vite.ResolvedConfig;
 
-	globalThis.astroAsset = {};
+	globalThis.astroAsset = {
+		referencedImages: new Set(),
+	};
 
 	return [
 		// Expose the components and different utilities from `astro:assets` and handle serving images from `/_image` in dev
@@ -81,22 +84,28 @@ export default function assets({
 					if (!globalThis.astroAsset.staticImages) {
 						globalThis.astroAsset.staticImages = new Map<
 							string,
-							Map<string, { finalPath: string; transform: ImageTransform }>
+							{
+								originalSrcPath: string;
+								transforms: Map<string, { finalPath: string; transform: ImageTransform }>;
+							}
 						>();
 					}
 
-					const originalImagePath = (
+					// Rollup will copy the file to the output directory, this refer to this final path, not to the original path
+					const finalOriginalImagePath = (
 						isESMImportedImage(options.src) ? options.src.src : options.src
 					).replace(settings.config.build.assetsPrefix || '', '');
-					const hash = hashTransform(
-						options,
-						settings.config.image.service.entrypoint,
-						hashProperties
-					);
+
+					// This, however, is the real original path, in `src` and all.
+					const originalSrcPath = isESMImportedImage(options.src)
+						? options.src.fsPath
+						: options.src;
+
+					const hash = hashTransform(options, settings.config.image.service.entrypoint, hashProperties);
 
 					let finalFilePath: string;
-					let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath);
-					let transformForHash = transformsForPath?.get(hash);
+					let transformsForPath = globalThis.astroAsset.staticImages.get(finalOriginalImagePath);
+					let transformForHash = transformsForPath?.transforms.get(hash);
 					if (transformsForPath && transformForHash) {
 						finalFilePath = transformForHash.finalPath;
 					} else {
@@ -105,11 +114,17 @@ export default function assets({
 						);
 
 						if (!transformsForPath) {
-							globalThis.astroAsset.staticImages.set(originalImagePath, new Map());
-							transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!;
+							globalThis.astroAsset.staticImages.set(finalOriginalImagePath, {
+								originalSrcPath: originalSrcPath,
+								transforms: new Map(),
+							});
+							transformsForPath = globalThis.astroAsset.staticImages.get(finalOriginalImagePath)!;
 						}
 
-						transformsForPath.set(hash, { finalPath: finalFilePath, transform: options });
+						transformsForPath.transforms.set(hash, {
+							finalPath: finalFilePath,
+							transform: options,
+						});
 					}
 
 					if (settings.config.build.assetsPrefix) {
@@ -171,7 +186,8 @@ export default function assets({
 						});
 					}
 
-					return `export default ${JSON.stringify(meta)}`;
+					return `
+					export default ${getProxyCode(meta, isServerLikeOutput(settings.config))}`;
 				}
 			},
 		},

packages/astro/src/content/runtime-assets.ts

@@ -22,7 +22,7 @@ export function createImage(pluginContext: PluginContext, entryFilePath: string)
 				return z.never();
 			}
 
-			return metadata;
+			return { ...metadata, ASTRO_ASSET: true };
 		});
 	};
 }

packages/astro/src/content/vite-plugin-content-imports.ts

@@ -12,8 +12,10 @@ import type {
 	DataEntryModule,
 	DataEntryType,
 } from '../@types/astro.js';
+import { getProxyCode } from '../assets/utils/proxy.js';
 import { AstroError } from '../core/errors/errors.js';
 import { AstroErrorData } from '../core/errors/index.js';
+import { isServerLikeOutput } from '../prerender/utils.js';
 import { escapeViteEnvReferences } from '../vite-plugin-utils/index.js';
 import { CONTENT_FLAG, DATA_FLAG } from './consts.js';
 import {
@@ -94,7 +96,7 @@ export function astroContentImportPlugin({
 					const code = escapeViteEnvReferences(`
 export const id = ${JSON.stringify(id)};
 export const collection = ${JSON.stringify(collection)};
-export const data = ${stringifyEntryData(data)};
+export const data = ${stringifyEntryData(data, isServerLikeOutput(settings.config))};
 export const _internal = {
 	type: 'data',
 	filePath: ${JSON.stringify(_internal.filePath)},
@@ -118,7 +120,7 @@ export const _internal = {
 						export const collection = ${JSON.stringify(collection)};
 						export const slug = ${JSON.stringify(slug)};
 						export const body = ${JSON.stringify(body)};
-						export const data = ${stringifyEntryData(data)};
+						export const data = ${stringifyEntryData(data, isServerLikeOutput(settings.config))};
 						export const _internal = {
 							type: 'content',
 							filePath: ${JSON.stringify(_internal.filePath)},
@@ -352,13 +354,19 @@ async function getContentConfigFromGlobal() {
 }
 
 /** Stringify entry `data` at build time to be used as a Vite module */
-function stringifyEntryData(data: Record<string, any>): string {
+function stringifyEntryData(data: Record<string, any>, isSSR: boolean): string {
 	try {
 		return devalue.uneval(data, (value) => {
 			// Add support for URL objects
 			if (value instanceof URL) {
 				return `new URL(${JSON.stringify(value.href)})`;
 			}
+
+			// For Astro assets, add a proxy to track references
+			if (typeof value === 'object' && 'ASTRO_ASSET' in value) {
+				const { ASTRO_ASSET, ...asset } = value;
+				return getProxyCode(asset, isSSR);
+			}
 		});
 	} catch (e) {
 		if (e instanceof Error) {

packages/astro/src/core/build/generate.ts

@@ -205,7 +205,7 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn
 		logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`);
 
 		const totalCount = Array.from(staticImageList.values())
-			.map((x) => x.size)
+			.map((x) => x.transforms.size)
 			.reduce((a, b) => a + b, 0);
 		const cpuCount = os.cpus().length;
 		const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount);

tsconfig.base.json

@@ -9,6 +9,7 @@
     "module": "Node16",
     "esModuleInterop": true,
     "skipLibCheck": true,
-    "verbatimModuleSyntax": true
+    "verbatimModuleSyntax": true,
+    "stripInternal": true
   }
 }