From 78b8b69fe4894b30c8a826d3549cdd06b56bf99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 10:41:06 +0200 Subject: [PATCH 1/7] [JSONBridge] New generic JSON bridge --- bridges/JSONBridge.php | 130 +++++++++++++++++++++++++++++++++++++++++ composer.json | 3 +- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 bridges/JSONBridge.php diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php new file mode 100644 index 00000000000..29834e4002f --- /dev/null +++ b/bridges/JSONBridge.php @@ -0,0 +1,130 @@ + [ + 'name' => 'JSON URL', + 'type' => 'text', + 'required' => true, + 'title' => 'The URL returning a JSON document (array or object)', + ], + 'root' => [ + 'name' => 'JMESPath: items selector', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '[*]', + 'title' => 'JMESPath expression selecting the list of items (usually "[*]" for top-level arrays)', + ], + 'id' => [ + 'name' => 'JMESPath (per item): ID', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'id', + 'title' => 'JMESPath expression for a unique ID per item)', + ], + 'uri' => [ + 'name' => 'JMESPath (per item): URL', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'thumb.url', + 'title' => 'JMESPath expression for the item link (e.g. a product or article URL)', + ], + 'title' => [ + 'name' => 'JMESPath (per item): Title', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'name', + 'title' => 'JMESPath expression for the item title (headline, product name, etc.)', + ], + 'content' => [ + 'name' => 'JMESPath (per item): Content/Description', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'join(``, [`Price: `, price.formatted, ` USD`])', + 'title' => 'Optional: JMESPath expression for description text. You can use join() to concatenate fields.', + ], + 'image' => [ + 'name' => 'JMESPath (per item): Image URL', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'thumb.url', + 'title' => 'Optional: JMESPath expression for an image URL to add as an enclosure', + ], + ]]; + + public function collectData() { + $raw = getContents($this->getInput('url')); + $json = json_decode($raw, true); + if ($json === null) { + returnServerError('Invalid JSON'); + } + + // root must be array or a single object + try { + $items = JmesPath::search($this->getInput('root'), $json); + } catch (\Throwable $e) { + returnServerError('JMESPath error in "root": ' . $e->getMessage()); + } + if (!is_array($items)) { + returnServerError('Root JMESPath must return an array or an object'); + } + + // If it's an associative array (single object), wrap it; if it's a list, use as-is + $list = array_is_list($items) ? $items : [$items]; + + foreach ($list as $idx => $it) { + if (!is_array($it)) { + returnServerError("Item #$idx is not an object/assoc array"); + } + + // Required + $id = $this->extract($it, $this->getInput('id'), 'id', $idx); + if ($id === null || $id === '') { + returnServerError("Required 'id' missing or not scalar for item #$idx"); + } + + $uri = $this->extract($it, $this->getInput('uri'), 'uri', $idx); + if ($uri === null || $uri === '') { + returnServerError("Required 'uri' missing or not scalar for item #$idx"); + } + + $title = $this->extract($it, $this->getInput('title'), 'title', $idx); + if ($title === null || $title === '') { + returnServerError("Required 'title' missing or not scalar for item #$idx"); + } + + // Optional + $content = $this->extract($it, $this->getInput('content'), 'content', $idx); + $image = $this->extract($it, $this->getInput('image'), 'image', $idx); + + $this->items[] = [ + 'uid' => (string)$id, + 'uri' => (string)$uri, + 'title' => (string)$title, + 'content' => $content ?? '', + 'enclosures' => $image ? [$image . '#.image'] : [], + ]; + } + } + + /** + * Single helper: evaluate a JMESPath expression. + * - Throws server error on JMESPath syntax/runtime errors. + * - Returns scalar string or null (if expr empty or result non-scalar/empty). + */ + private function extract(array $context, ?string $expr, string $param, int $idx): ?string { + if (!$expr) return null; // caller decides if required + try { + $res = JmesPath::search($expr, $context); + } catch (\Throwable $e) { + returnServerError("JMESPath error in '$param' for item #$idx: " . $e->getMessage()); + } + return (is_scalar($res) && $res !== '') ? (string)$res : null; + } +} \ No newline at end of file diff --git a/composer.json b/composer.json index 8d41c51b889..024581235ef 100644 --- a/composer.json +++ b/composer.json @@ -30,7 +30,8 @@ "ext-simplexml": "*", "ext-dom": "*", "ext-json": "*", - "ext-filter": "*" + "ext-filter": "*", + "mtdowling/jmespath.php": "*" }, "require-dev": { "phpunit/phpunit": "^9", From 5d8bbb132db788d349b7945101b9fd4353905541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 11:16:12 +0200 Subject: [PATCH 2/7] [JSONBridge] Add cookie param --- bridges/JSONBridge.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php index 29834e4002f..e6e62f30b34 100644 --- a/bridges/JSONBridge.php +++ b/bridges/JSONBridge.php @@ -12,8 +12,14 @@ class JSONBridge extends BridgeAbstract { 'name' => 'JSON URL', 'type' => 'text', 'required' => true, + 'exampleValue' => 'https://example.com/json/', 'title' => 'The URL returning a JSON document (array or object)', ], + 'cookie' => [ + 'name' => 'The complete cookie value', + 'title' => 'Paste the cookie value from your browser if needed', + 'required' => false, + ], 'root' => [ 'name' => 'JMESPath: items selector', 'type' => 'text', @@ -59,7 +65,13 @@ class JSONBridge extends BridgeAbstract { ]]; public function collectData() { - $raw = getContents($this->getInput('url')); + if ($this->getInput('cookie')) { + $raw = getContents($this->getInput('url'), [], [CURLOPT_COOKIE => $this->getInput('cookie')]); + } + else { + $raw = getContents($this->getInput('url')); + } + $json = json_decode($raw, true); if ($json === null) { returnServerError('Invalid JSON'); From d250604e21262b5ece732af3d54650a52a5cda8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 11:21:41 +0200 Subject: [PATCH 3/7] [JSONBridge] use real example URL --- bridges/JSONBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php index e6e62f30b34..e5098d1c73a 100644 --- a/bridges/JSONBridge.php +++ b/bridges/JSONBridge.php @@ -12,7 +12,7 @@ class JSONBridge extends BridgeAbstract { 'name' => 'JSON URL', 'type' => 'text', 'required' => true, - 'exampleValue' => 'https://example.com/json/', + 'exampleValue' => 'https://registry.npmjs.org/react/latest', 'title' => 'The URL returning a JSON document (array or object)', ], 'cookie' => [ From 3df2cb36b3822b0c1e0bbf31a16234cc06f098c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 11:25:44 +0200 Subject: [PATCH 4/7] [JSONBridge] fix issues reportd by PHPCS --- bridges/JSONBridge.php | 47 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php index e5098d1c73a..22e9ea707c0 100644 --- a/bridges/JSONBridge.php +++ b/bridges/JSONBridge.php @@ -1,13 +1,15 @@ [ 'name' => 'JSON URL', 'type' => 'text', @@ -64,27 +66,27 @@ class JSONBridge extends BridgeAbstract { ], ]]; - public function collectData() { + public function collectData() + { if ($this->getInput('cookie')) { $raw = getContents($this->getInput('url'), [], [CURLOPT_COOKIE => $this->getInput('cookie')]); - } - else { + } else { $raw = getContents($this->getInput('url')); } $json = json_decode($raw, true); if ($json === null) { - returnServerError('Invalid JSON'); + throwServerException('Invalid JSON'); } // root must be array or a single object try { $items = JmesPath::search($this->getInput('root'), $json); } catch (\Throwable $e) { - returnServerError('JMESPath error in "root": ' . $e->getMessage()); + throwServerException('JMESPath error in "root": ' . $e->getMessage()); } if (!is_array($items)) { - returnServerError('Root JMESPath must return an array or an object'); + throwServerException('Root JMESPath must return an array or an object'); } // If it's an associative array (single object), wrap it; if it's a list, use as-is @@ -92,35 +94,35 @@ public function collectData() { foreach ($list as $idx => $it) { if (!is_array($it)) { - returnServerError("Item #$idx is not an object/assoc array"); + throwServerException("Item #$idx is not an object/assoc array"); } // Required $id = $this->extract($it, $this->getInput('id'), 'id', $idx); if ($id === null || $id === '') { - returnServerError("Required 'id' missing or not scalar for item #$idx"); + throwServerException("Required 'id' missing or not scalar for item #$idx"); } $uri = $this->extract($it, $this->getInput('uri'), 'uri', $idx); if ($uri === null || $uri === '') { - returnServerError("Required 'uri' missing or not scalar for item #$idx"); + throwServerException("Required 'uri' missing or not scalar for item #$idx"); } $title = $this->extract($it, $this->getInput('title'), 'title', $idx); if ($title === null || $title === '') { - returnServerError("Required 'title' missing or not scalar for item #$idx"); + throwServerException("Required 'title' missing or not scalar for item #$idx"); } // Optional $content = $this->extract($it, $this->getInput('content'), 'content', $idx); - $image = $this->extract($it, $this->getInput('image'), 'image', $idx); + $image = $this->extract($it, $this->getInput('image'), 'image', $idx); $this->items[] = [ - 'uid' => (string)$id, - 'uri' => (string)$uri, - 'title' => (string)$title, - 'content' => $content ?? '', - 'enclosures' => $image ? [$image . '#.image'] : [], + 'uid' => (string)$id, + 'uri' => (string)$uri, + 'title' => (string)$title, + 'content' => $content ?? '', + 'enclosures' => $image ? [$image . '#.image'] : [], ]; } } @@ -130,12 +132,15 @@ public function collectData() { * - Throws server error on JMESPath syntax/runtime errors. * - Returns scalar string or null (if expr empty or result non-scalar/empty). */ - private function extract(array $context, ?string $expr, string $param, int $idx): ?string { - if (!$expr) return null; // caller decides if required + private function extract(array $context, ?string $expr, string $param, int $idx): ?string + { + if (!$expr) { + return null; // caller decides if required + } try { $res = JmesPath::search($expr, $context); } catch (\Throwable $e) { - returnServerError("JMESPath error in '$param' for item #$idx: " . $e->getMessage()); + throwServerException("JMESPath error in '$param' for item #$idx: " . $e->getMessage()); } return (is_scalar($res) && $res !== '') ? (string)$res : null; } From 9075dd6c795776626311182e857ecdd7b7ae6367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 11:29:37 +0200 Subject: [PATCH 5/7] [JSONBridge] final touches --- bridges/JSONBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php index 22e9ea707c0..6425078373a 100644 --- a/bridges/JSONBridge.php +++ b/bridges/JSONBridge.php @@ -41,7 +41,7 @@ class JSONBridge extends BridgeAbstract 'type' => 'text', 'required' => true, 'exampleValue' => 'thumb.url', - 'title' => 'JMESPath expression for the item link (e.g. a product or article URL)', + 'title' => 'JMESPath expression for the item link (e.g. a product or article URL). Use join() to build the URL if needed.', ], 'title' => [ 'name' => 'JMESPath (per item): Title', From 50f29a7cbaaecafc03543abf43234c2b011f6df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 12:09:19 +0200 Subject: [PATCH 6/7] [JSONBridge] install jmespath.php in Dockerfile --- Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fb78334424d..2389aaec8f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,8 @@ RUN set -xe && \ curl \ # for patching libcurl-impersonate patchelf \ + # for installing jmespath.php + composer \ && \ # install curl-impersonate library curlimpersonate_version=1.0.0rc2 && \ @@ -53,7 +55,11 @@ RUN set -xe && \ tar xaf "$archive" -C /usr/local/lib/curl-impersonate && \ patchelf --set-soname libcurl.so.4 /usr/local/lib/curl-impersonate/libcurl-impersonate.so && \ rm "$archive" && \ - apt-get purge --assume-yes curl patchelf && \ + # install jmespath.php using composer + mkdir /app && composer require mtdowling/jmespath.php --no-scripts --no-plugins --no-interaction --working-dir=/app && chown -R --no-dereference www-data:www-data /app && \ + # cleanup buildtime dependencies + composer clear-cache && \ + apt-get purge --assume-yes curl patchelf composer && \ rm -rf /var/lib/apt/lists/* ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate.so From 53db1a65eb6d8e589696bb31d53577ea0b3125fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wro=CC=81bel?= Date: Wed, 3 Sep 2025 14:49:43 +0200 Subject: [PATCH 7/7] [JSONBridge] Add Feed Title and URI --- bridges/JSONBridge.php | 43 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/bridges/JSONBridge.php b/bridges/JSONBridge.php index 6425078373a..70fccab5a9f 100644 --- a/bridges/JSONBridge.php +++ b/bridges/JSONBridge.php @@ -22,6 +22,20 @@ class JSONBridge extends BridgeAbstract 'title' => 'Paste the cookie value from your browser if needed', 'required' => false, ], + 'feed_uri' => [ + 'name' => 'JMESPath: feed\'s base URI', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'join(``, search_query, `.html`])', + 'title' => 'Optional: JMESPath expression for the feed\'s own URI, e.g. a HTML page that renders the JSON. Corresponds to getURI() function.', + ], + 'feed_name' => [ + 'name' => 'JMESPath: feed\'s name', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'search_query', + 'title' => 'Optional: JMESPath expression for the feed\'s title. Corresponds to getName() function.', + ], 'root' => [ 'name' => 'JMESPath: items selector', 'type' => 'text', @@ -34,13 +48,13 @@ class JSONBridge extends BridgeAbstract 'type' => 'text', 'required' => true, 'exampleValue' => 'id', - 'title' => 'JMESPath expression for a unique ID per item)', + 'title' => 'JMESPath expression for a unique ID per item.', ], 'uri' => [ 'name' => 'JMESPath (per item): URL', 'type' => 'text', 'required' => true, - 'exampleValue' => 'thumb.url', + 'exampleValue' => 'item.url', 'title' => 'JMESPath expression for the item link (e.g. a product or article URL). Use join() to build the URL if needed.', ], 'title' => [ @@ -58,7 +72,7 @@ class JSONBridge extends BridgeAbstract 'title' => 'Optional: JMESPath expression for description text. You can use join() to concatenate fields.', ], 'image' => [ - 'name' => 'JMESPath (per item): Image URL', + 'name' => 'JMESPath (per item): Image URI', 'type' => 'text', 'required' => false, 'exampleValue' => 'thumb.url', @@ -66,6 +80,27 @@ class JSONBridge extends BridgeAbstract ], ]]; + private $name; + private $feed_uri; + + public function getName() + { + if (isset($this->name)) { + return $this->name; + } + + return parent::getName(); + } + + public function getURI() + { + if (isset($this->feed_uri)) { + return $this->feed_uri; + } + + return parent::getURI(); + } + public function collectData() { if ($this->getInput('cookie')) { @@ -116,6 +151,8 @@ public function collectData() // Optional $content = $this->extract($it, $this->getInput('content'), 'content', $idx); $image = $this->extract($it, $this->getInput('image'), 'image', $idx); + $name = $this->extract($it, $this->getInput('name'), 'name', $idx); + $feed_uri = $this->extract($it, $this->getInput('feed_uri'), 'feed_uri', $idx); $this->items[] = [ 'uid' => (string)$id,