build: download and extract micro:bit hex during npm install

Author: cwillisf

.eslintrc.js

@@ -1,3 +1,3 @@
 module.exports = {
-    extends: ['scratch', 'scratch/node']
+    extends: ['scratch', 'scratch/node', 'scratch/es6']
 };

.gitignore

@@ -17,3 +17,9 @@ npm-*
 # Generated translation files
 /translations
 /locale
+
+# Other generated source
+/src/generated
+
+# Downloaded during "npm install"
+/static/microbit

package-lock.json

@@ -14,6 +14,7 @@
     "build": "npm run clean && webpack --colors --bail",
     "clean": "rimraf ./build && mkdirp build && rimraf ./dist && mkdirp dist",
     "deploy": "touch build/.nojekyll && gh-pages -t -d build -m \"[skip ci] Build for $(git log --pretty=format:%H -n1)\"",
+    "install": "node scripts/install.mjs",
     "prepare": "husky install",
     "prune": "./prune-gh-pages.sh",
     "i18n:push": "tx-push-src scratch-editor interface translations/en.json",
@@ -139,7 +140,8 @@
     "web-audio-test-api": "0.5.2",
     "webpack": "4.46.0",
     "webpack-cli": "3.3.12",
-    "webpack-dev-server": "3.11.2"
+    "webpack-dev-server": "3.11.2",
+    "yauzl": "2.10.0"
   },
   "jest": {
     "setupFiles": [

package.json

@@ -0,0 +1,8 @@
+const path = require('path');
+module.exports = {
+    extends: [path.resolve(__dirname, '..', '.eslintrc.js')],
+    rules: {
+        // NPM scripts are allowed to use console.log & friends
+        'no-console': 'off'
+    }
+};

scripts/.eslintrc.js

@@ -0,0 +1,121 @@
+// Install any non-NPM dependencies in this script.
+// This script is for downloading dependencies, not for building.
+// Once the "install" step is complete, a developer should be able to work without an Internet connection.
+// See also: https://docs.npmjs.com/cli/using-npm/scripts
+
+import fs from 'fs';
+import path from 'path';
+
+import crossFetch from 'cross-fetch';
+import yauzl from 'yauzl';
+import {fileURLToPath} from 'url';
+
+/** @typedef {import('yauzl').Entry} ZipEntry */
+/** @typedef {import('yauzl').ZipFile} ZipFile */
+
+// these aren't set in ESM mode
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// base/root path for the project
+const basePath = path.join(__dirname, '..');
+
+/**
+ * Extract the first matching file from a zip buffer.
+ * The path within the zip file is ignored: the destination path is `${destinationDirectory}/${basename(entry.name)}`.
+ * Prints warnings if more than one matching file is found.
+ * @param {function(ZipEntry): boolean} filter Returns true if the entry should be extracted.
+ * @param {string} relativeDestDir The directory to extract to, relative to `basePath`.
+ * @param {Buffer} zipBuffer A buffer containing the zip file.
+ * @returns {Promise<string>} A Promise for the base name of the written file (without directory).
+ */
+const extractFirstMatchingFile = (filter, relativeDestDir, zipBuffer) => new Promise((resolve, reject) => {
+    try {
+        let extractedFileName;
+        yauzl.fromBuffer(zipBuffer, {lazyEntries: true}, (zipError, zipfile) => {
+            if (zipError) {
+                throw zipError;
+            }
+            zipfile.readEntry();
+            zipfile.on('end', () => {
+                resolve(extractedFileName);
+            });
+            zipfile.on('entry', entry => {
+                if (!filter(entry)) {
+                    // ignore non-matching file
+                    return zipfile.readEntry();
+                }
+                if (extractedFileName) {
+                    console.warn(`Multiple matching files found. Ignoring: ${entry.fileName}`);
+                    return zipfile.readEntry();
+                }
+                extractedFileName = entry.fileName;
+                console.info(`Found matching file: ${entry.fileName}`);
+                zipfile.openReadStream(entry, (fileError, readStream) => {
+                    if (fileError) {
+                        throw fileError;
+                    }
+                    const baseName = path.basename(entry.fileName);
+                    const relativeDestFile = path.join(relativeDestDir, baseName);
+                    console.info(`Extracting ${relativeDestFile}`);
+                    const absoluteDestDir = path.join(basePath, relativeDestDir);
+                    fs.mkdirSync(absoluteDestDir, {recursive: true});
+                    const absoluteDestFile = path.join(basePath, relativeDestFile);
+                    const outStream = fs.createWriteStream(absoluteDestFile);
+                    readStream.on('end', () => {
+                        outStream.close();
+                        zipfile.readEntry();
+                    });
+                    readStream.pipe(outStream);
+                });
+            });
+        });
+    } catch (error) {
+        reject(error);
+    }
+});
+
+const downloadMicrobitHex = async () => {
+    const url = 'https://downloads.scratch.mit.edu/microbit/scratch-microbit.hex.zip';
+    console.info(`Downloading ${url}`);
+    const response = await crossFetch(url);
+    const zipBuffer = Buffer.from(await response.arrayBuffer());
+    const relativeHexDir = path.join('static', 'microbit');
+    const hexFileName = await extractFirstMatchingFile(
+        entry => /\.hex$/.test(entry.fileName),
+        path.join('static', 'microbit'),
+        zipBuffer
+    );
+    const relativeHexFile = path.join(relativeHexDir, hexFileName);
+    const relativeGeneratedDir = path.join('src', 'generated');
+    const relativeGeneratedFile = path.join(relativeGeneratedDir, 'microbit-hex-url.cjs');
+    const absoluteGeneratedDir = path.join(basePath, relativeGeneratedDir);
+    fs.mkdirSync(absoluteGeneratedDir, {recursive: true});
+    const absoluteGeneratedFile = path.join(basePath, relativeGeneratedFile);
+    fs.writeFileSync(
+        absoluteGeneratedFile,
+        [
+            '// This file is generated by scripts/install.mjs',
+            '// Do not edit this file directly',
+            '// This file relies on a loader to turn this `require` into a URL',
+            `module.exports = require('./${path.relative(relativeGeneratedDir, relativeHexFile)}');`,
+            '' // final newline
+        ].join('\n')
+    );
+    console.info(`Wrote ${relativeGeneratedFile}`);
+};
+
+const install = async () => {
+    await downloadMicrobitHex();
+};
+
+install().then(
+    () => {
+        console.info('Install script complete');
+        process.exit(0);
+    },
+    e => {
+        console.error(e);
+        process.exit(1);
+    }
+);