Add: retry on failure (#273)

* Bump deps

* Add: retry on failure

* Use retry

Author: eight04

README.md

@@ -118,6 +118,27 @@ pbs.twimg.com/media/$1:orig
 * Lines starting with `#` are ignored.
 * Empty lines are ignored.
 
+Retry on failure
+----------------
+
+Some sites return an error when getting too much traffic. You can make the extension wait some time and try again by defining retry rules:
+
+```
+# example
+example\.com/images/.*
+3 5 1
+```
+
+* Each rule includes:
+	- A line of regex matching the URL.
+	- A line of numbers, representing `maxRetries`, `delay`, and `exponentialBackoff`.
+* `maxRetries` - the extension will retry `maxRetries` times before raising a network error.
+* `delay` - after an error occurs, wait `delay` seconds before the next try.
+* `exponentialBackoff` - increase the delay gradually. For example, when `delay` is `5` and `exponentialBackoff` is `2`, the first delay will be 5s, the second will be 10s, the third will be 20s, etc.
+* Lines starting with `#` are ignored.
+* Empty lines are ignored.
+
+
 Domain blacklist
 ----------------
 

package-lock.json

@@ -24,12 +24,12 @@
   "devDependencies": {
     "@eight04/idb-storage": "^0.4.2",
     "content-disposition": "^0.5.3",
-    "eslint": "^7.32.0",
+    "eslint": "^8.1.0",
     "espression": "^1.8.5",
-    "mime": "^2.5.2",
+    "mime": "^3.0.0",
     "mkdoc": "^1.0.49",
     "node-sigint": "0.0.1",
-    "rollup": "^2.55.1",
+    "rollup": "^2.59.0",
     "rollup-plugin-cjs-es": "^1.0.1",
     "rollup-plugin-copy": "^3.4.0",
     "rollup-plugin-iife": "^0.5.0",
@@ -42,7 +42,7 @@
     "stream-throttle": "^0.1.3",
     "sync-version": "^1.0.1",
     "tiny-glob": "^0.2.9",
-    "web-ext": "^6.2.0",
+    "web-ext": "^6.5.0",
     "webext-menus": "^0.3.2",
     "webext-pref": "^0.6.0",
     "webext-pref-ui": "^0.2.1",

package.json

@@ -4,6 +4,7 @@ import browser from "webextension-polyfill";
 import {IS_CHROME} from "./env.js";
 import {fetchImage} from "./fetch-image.js";
 import {fetchXHR} from "./fetch.js";
+import {retry} from "./retry.js";
 
 export const imageCache = createImageCache();
 
@@ -26,7 +27,7 @@ function createImageCache() {
 
   function add({url, tabId, frameId, referrer}) {
     return cache.set(url, async () => {
-      const data = await _fetchImage(url, tabId, frameId, referrer);
+      const data = await retry(() => _fetchImage(url, tabId, frameId, referrer), url);
       const resource = data.blob;
       delete data.blob;
       const meta = Object.assign(data, await detectDimension(resource));

src/lib/image-cache.js

@@ -40,6 +40,7 @@ const DEFAULT = {
   srcAlternative: "data-src, data-gifsrc, gifsrc",
   previewMaxHeight: 200,
   previewMaxHeightUpperBound: 200,
+  retryOnFailure: "",
   urlMap: "",
   useCache: true,
   useExpression: false,

src/lib/pref.js

@@ -0,0 +1,7 @@
+export function *parseText(text, groupBy = 1) {
+  const lines = text.split(/\r?\n/g).filter(line =>
+    line && /\S/.test(line) && !line.startsWith("#"));
+  for (let i = 0; i < lines.length; i += groupBy) {
+    yield lines.slice(i, i + groupBy);
+  }
+}

src/lib/readline.js

@@ -0,0 +1,48 @@
+import {pref} from "./pref.js";
+import {parseText} from "./readline.js";
+import {timeout} from './timeout.js';
+
+let rules = [];
+
+pref.ready().then(() => {
+  update();
+  pref.on("change", change => {
+    if (change.retryOnFailure != null) {
+      update();
+    }
+  });
+});
+
+function update() {
+  rules = [...parseText(pref.get('retryOnFailure'), 2)]
+    .map(createRule);
+}
+
+function createRule(lines) {
+  const rx = new RegExp(lines[0], 'i');
+  const args = lines[1].split(/\s+/).map(Number);
+  return {
+    rx,
+    max: args[0],
+    delay: args[1],
+    exp: args[2]
+  };
+}
+
+export async function retry(fn, key) {
+  const c = rules.find(r => r.rx.test(key));
+  if (c) {
+    let delay = c.delay;
+    for (let i = 0; i < c.max; i++) {
+      try {
+        return await fn();
+      } catch (err) {
+        console.warn(err);
+        console.warn(`try again after ${delay}s`);
+      }
+      await timeout(delay * 1000);
+      delay *= c.exp;
+    }
+  }
+  return await fn();
+}

src/lib/retry.js

@@ -0,0 +1,3 @@
+export function timeout(t) {
+  return new Promise(resolve => setTimeout(resolve, t));
+}

src/lib/timeout.js

@@ -1,4 +1,5 @@
 import {pref} from "./pref.js";
+import {parseText} from "./readline.js";
 
 let transforms = [];
 
@@ -12,13 +13,8 @@ pref.ready().then(() => {
 });
 
 function update() {
-  const lines = pref.get("urlMap").split(/\r?\n/g).filter(line =>
-    line && /\S/.test(line) && !line.startsWith("#"));
-  const newTransforms = [];
-  for (let i = 0; i < lines.length; i += 2) {
-    newTransforms.push(createTransform(lines[i], lines[i + 1]));
-  }
-  transforms = newTransforms;
+  transforms = [...parseText(pref.get("urlMap"), 2)]
+    .map(lines => createTransform(...lines));
 }
 
 function createTransform(search, repl) {

src/lib/url-map.js

@@ -313,7 +313,14 @@ const root = createUI({
           label: _("optionUrlMapLabel"),
           learnMore: "https://github.com/eight04/image-picka#transform-url-with-regexp",
           className: "form-monospace"
-        },      
+        },
+        {
+          type: "textarea",
+          key: "retryOnFailure",
+          label: _("optionRetryOnFailureLabel"),
+          learnMore: "https://github.com/eight04/image-picka#retry-on-failure",
+          className: "form-monospace"
+        },
         {
           type: "textarea",
           key: "blacklist",

src/options.js

@@ -291,6 +291,10 @@
 			}
 		}
 	},
+  "optionRetryOnFailureLabel": {
+    "message": "Retry on failure",
+    "description": "Label of retryOnFailure option"
+  },
 	"optionUrlMapLabel": {
 		"message": "Transform URL with regexp",
 		"description": "Label of urlMap option"

src/static/_locales/en/messages.json

@@ -6,14 +6,28 @@ const fs = require("fs");
 const {parse: urlParse} = require("url");
 const {Throttle} = require("stream-throttle");
 const mime = require("mime");
+let i = 1;
 const HANDLES = [
 	{
 		test: /^throttle\.png/,
 		handle(req, res) {
 			const image = fs.createReadStream(__dirname + "/test.png");
 			image.pipe(new Throttle({rate: 100})).pipe(res);
 		}
-	}
+	},
+  {
+    test: /503\.png/,
+    handle(req, res) {
+      res.setHeader("Cache-Control", "no-cache");
+      if (i++ % 10) {
+        res.writeHead(503);
+        res.end('503');
+        return;
+      }
+			const image = fs.createReadStream(__dirname + "/test.png");
+			image.pipe(res);
+    }
+  }
 ];
 const server = http.createServer((req, res) => {
 	console.log(`${new Date} ${req.method} ${req.url}`);