From 6ff8aa3c5255478e91b0b533be7d1dde6510f47f Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Thu, 4 Sep 2025 12:03:36 +0200 Subject: [PATCH 001/254] feat: adjust to changed data model; part 1 - add places in the template for (some of) relocated elements - change form bindings for the Object section --- src/templates/edit.html | 8 ++- src/templates/fore/epidoc-template.xml | 81 +++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/src/templates/edit.html b/src/templates/edit.html index a6d1027..42cb365 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -453,7 +453,7 @@

- + @@ -471,8 +471,12 @@

+ + + + - + diff --git a/src/templates/fore/epidoc-template.xml b/src/templates/fore/epidoc-template.xml index ae2ba4a..75f2e4d 100644 --- a/src/templates/fore/epidoc-template.xml +++ b/src/templates/fore/epidoc-template.xml @@ -9,6 +9,10 @@ + + + + Epigraphische Datenbank Heidelberg @@ -21,23 +25,65 @@ - Epigraphische Datenbank Heidelberg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -51,7 +97,23 @@ @datingMethod - fixes Attribut, hier immer Julianisch --> + + + + + + + + @@ -166,6 +228,23 @@ + + + + + + + + + + + + + + + + + From 14fb3aa796c4676931ae4fdf4065394c0f824d8f Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Thu, 4 Sep 2025 13:11:33 +0200 Subject: [PATCH 002/254] feat: remove all msPart stuff from epidoc-template --- src/templates/fore/epidoc-template.xml | 115 +------------------------ 1 file changed, 3 insertions(+), 112 deletions(-) diff --git a/src/templates/fore/epidoc-template.xml b/src/templates/fore/epidoc-template.xml index 75f2e4d..a756466 100644 --- a/src/templates/fore/epidoc-template.xml +++ b/src/templates/fore/epidoc-template.xml @@ -111,122 +111,13 @@ --> - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + From c6b424345dba01f59088571300e71c047a7dddfb Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Thu, 4 Sep 2025 13:42:12 +0200 Subject: [PATCH 003/254] feat: remove processing related to creation of msParts - remove form template mspart-tmpl; no longer used - remove tei:msPart from path expressions where content was deliberately pulled (keep the rest, may need revision) - remove handling of msDesc which was explicitely pulling only specific content and then wrapping parts of it in msPart - retain the scaffolding of api:to-ms-part function but only keep div handling bits (may need further revision, but dropping this off altogether caused complete breakdown of the form) --- src/modules/custom-api.xql | 85 ++------------------- src/templates/fore/mspart-tmpl.xml | 115 ----------------------------- 2 files changed, 7 insertions(+), 193 deletions(-) delete mode 100644 src/templates/fore/mspart-tmpl.xml diff --git a/src/modules/custom-api.xql b/src/modules/custom-api.xql index bdb0174..49d516a 100644 --- a/src/modules/custom-api.xql +++ b/src/modules/custom-api.xql @@ -379,12 +379,7 @@ declare function api:inscription-template($request as map(*)) { collection($collection)//tei:idno[. = $id]/ancestor::tei:TEI, doc($collection || "/" || $id || ".xml")/tei:TEI )[1] - let $withParts := - if ($input//tei:msPart[@type="main"]) then - $input - else - document { api:to-ms-part($input) } - let $merged := api:file-upload(doc($config:inscription-templ), root($withParts)) + let $merged := api:file-upload(doc($config:inscription-templ), root(document {api:to-ms-part($input)})) return $merged else @@ -432,15 +427,15 @@ declare %private function api:postprocess($nodes as node()*, $edepId as xs:strin element { node-name($node) } { $node/@*, api:postprocess($node/tei:teiHeader, $edepId), - root($node)//tei:msPart/tei:facsimile, + root($node)//tei:facsimile, api:postprocess($node/tei:text, $edepId) } case element(tei:body) return element { node-name($node) } { $node/@*, - root($node)//tei:msPart/tei:div[@type=('apparatus', 'translation')], + root($node)//tei:div[@type=('apparatus', 'translation')],
- { root($node)//tei:msPart/tei:div[@type='textpart'] } + { root($node)//tei:div[@type='textpart'] }
, $node/tei:div[@type = "commentary"] } @@ -547,7 +542,7 @@ declare function api:render($request as map(*)) { let $xml := switch ($type) case "transcription" return - $request?body//tei:msPart/tei:div[@type="textpart"] + $request?body//tei:div[@type="textpart"] default return $request?body return @@ -570,72 +565,6 @@ declare function api:to-ms-part($nodes as node()*) { for $node in $nodes return typeswitch ($node) - case element(tei:msDesc) return - element { node-name($node) } { - $node/tei:msIdentifier, - if ($node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support) then - - - - - { - let $supp := $node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support - return ( - $supp/tei:objectType, - $supp/tei:material, - $supp/tei:note - ) - } - - - - - else - (), - if ($node/tei:history/tei:origin/tei:origDate) then - - - { $node/tei:history/tei:origin/tei:origDate } - - - else - (), - - - - - { $node/tei:msContents } - { - if ($node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support) then - - - - - { - $node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support/tei:dimensions, - $node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support/tei:rs - } - - { $node/tei:physDesc/tei:objectDesc/tei:supportDesc/tei:support/tei:condition } - - { - $node/tei:physDesc/tei:objectDesc/tei:layoutDesc - } - - - else - () - } - { - if ($node/tei:history/tei:provenance) then - - { $node/tei:history/tei:provenance } - - else - () - } - - } case element(tei:div) return if ($node/@type = "edition" and not($node/tei:div[@type='textpart'])) then
@@ -718,9 +647,9 @@ declare %private function api:complete-input($nodes as node()*) as node()* { { ($node/@type, $templateBibl/@type)[1] } {($node/node(), $templateBibl/*[not(local-name() = $node/node()/local-name())])} - case element(tei:msPart) return + (: case element(tei:msPart) return let $templateMsPart := doc('/db/apps/edep/templates/fore/mspart-tmpl.xml')/tei:msPart - return api:process-additional-template($templateMsPart, $node) + return api:process-additional-template($templateMsPart, $node) :) default return $node }; diff --git a/src/templates/fore/mspart-tmpl.xml b/src/templates/fore/mspart-tmpl.xml deleted file mode 100644 index b0c4656..0000000 --- a/src/templates/fore/mspart-tmpl.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - -
- -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - complete - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From a1a8d96cc5f849d02a6814e7b259303bddb03169 Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Fri, 5 Sep 2025 10:15:44 +0200 Subject: [PATCH 004/254] feat: adjust template to changed data model - type of inscription and note as msContents/@class and msContents/summary - add discussion of dating - reorder various types of bibliography as specified in the meeting - adjust handling of main and used languages (via langUsage) --- src/templates/fore/epidoc-template.xml | 41 ++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/templates/fore/epidoc-template.xml b/src/templates/fore/epidoc-template.xml index a756466..ea1fc23 100644 --- a/src/templates/fore/epidoc-template.xml +++ b/src/templates/fore/epidoc-template.xml @@ -33,6 +33,9 @@ + + + @@ -88,6 +91,31 @@ - - - - - @@ -131,6 +153,9 @@ + + + @@ -152,6 +177,10 @@ + + + + From c4e52405015278cbc89d113b53024cda7d808275 Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Fri, 5 Sep 2025 10:20:08 +0200 Subject: [PATCH 005/254] fix: cut out some msPart-specific bits, now superfluous --- src/templates/edit.html | 141 ++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 85 deletions(-) diff --git a/src/templates/edit.html b/src/templates/edit.html index 42cb365..651225f 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -119,10 +119,6 @@ - - - - @@ -324,18 +320,19 @@ - + -

- Objekt -

+

Objekt

- - + + - + - - - - - - - - - - - - - - - - - - - - - - - - Ungültiges Datum - - -
+ > -
-
+
+ + + + + + Ungültiges + Datum + + +
+
+ + + + + + Ungültiges + Datum + + +
- - - - Ungültiges Datum - - -
- - - -
- - - + + + + + +
+ + +
Date: Wed, 24 Sep 2025 14:22:24 +0200 Subject: [PATCH 036/254] fix(xml-editor): do not repeat the toolbar for every xml editor Instead, reuse a webcomponent that defines it --- src/resources/scripts/edep-xml-editor.js | 115 ++++++ src/templates/edit.html | 500 ++--------------------- 2 files changed, 140 insertions(+), 475 deletions(-) create mode 100644 src/resources/scripts/edep-xml-editor.js diff --git a/src/resources/scripts/edep-xml-editor.js b/src/resources/scripts/edep-xml-editor.js new file mode 100644 index 0000000..23caafc --- /dev/null +++ b/src/resources/scripts/edep-xml-editor.js @@ -0,0 +1,115 @@ +/** + * @typedef {{label: string, title: string, snippet: string}} Snippet + */ + +/** + * @param {Snippet[]} snippets + */ +const makeToolbarHTML = snippets => { + return ` +
+ + + + + + + + ${snippets.map( + ({ title, label, snippet }) => ` ` + )} +
+ `; +}; + +/** + * A jinn-xml-editor preconfigured for EDEP + */ +class EdepXMLEditor extends HTMLElement { + constructor() { + super(); + /** + * @type {string} + */ + this.schemaRoot = ''; + /** + * @type {string} + */ + this.placeholder = ''; + + /** + * @type Snippet[] + */ + this.snippets = []; + + // Constants + this.schema = 'resources/scripts/tei.json'; + this.unwrap = 'unwrap'; + this.namespace = 'http://www.tei-c.org/ns/1.0'; + + /** + * @type {HTMLElement} + */ + this.jinnXMLEditor = null; + } + + set value(newValue) { + this.jinnXMLEditor.value = newValue; + } + + get value() { + return this.jinnXMLEditor.value; + } + + connectedCallback() { + this.schemaRoot = this.getAttribute('schema-root'); + this.placeholder = this.getAttribute('placeholder'); + this.snippets = this.hasAttribute('snippets') + ? JSON.parse(this.getAttribute('snippets')) + : [ + { + label: 'ref', + title: 'Insert reference', + snippet: + '<ref type="biblio" target="$|1|">$|_|</ref>', + }, + ]; + + this.innerHTML = `${makeToolbarHTML(this.snippets)}`; + + this.jinnXMLEditor = this.firstElementChild; + } +} + +window.customElements.define('edep-xml-editor', EdepXMLEditor); diff --git a/src/templates/edit.html b/src/templates/edit.html index 5ca57eb..37ed1cf 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -63,6 +63,10 @@ + - - + + @@ -142,29 +100,14 @@ unresolved="unresolved" require-language="" locales="resources/i18n/{{ns}}/{{lng}}.json" - url-ignore="selectors" - > - + url-ignore="selectors"> + - + - - - + + + @@ -175,12 +118,11 @@ src="document1" view="single" subscribe="transcription" - use-language="use-language" - > + use-language="use-language"> -
+
diff --git a/src/templates/pages/people.html b/src/templates/pages/people.html index 39dfb9d..b818ee5 100644 --- a/src/templates/pages/people.html +++ b/src/templates/pages/people.html @@ -1,60 +1,23 @@ + - - - - - - + + + + + + - + <title data-template="config:app-title"> - - + + - - + - + - - + +
- +
-
+
- + - + - - + +
- + @@ -129,13 +68,12 @@ subforms="#options" selected="A" emit="transcription" - subscribe="transcription" - /> + subscribe="transcription">
-
+
@@ -366,8 +344,8 @@ -
- +
- + Language(s) @@ -1049,7 +1029,7 @@

Inscription

-

+

Referenzen @@ -1057,7 +1037,7 @@

-

+

@@ -1683,7 +1663,7 @@

- + - +
From 09946f2f7405dea534d0c5887ef6482147d20249 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 2 Oct 2025 17:03:31 +0200 Subject: [PATCH 059/254] group padding --- src/resources/css/edep-theme2.css | 7 +++++++ src/templates/geo-picker.html | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/resources/css/edep-theme2.css b/src/resources/css/edep-theme2.css index c0160b4..9833f45 100644 --- a/src/resources/css/edep-theme2.css +++ b/src/resources/css/edep-theme2.css @@ -37,6 +37,10 @@ details{ margin-bottom:1em; padding-bottom:1em; } + +} +details > fx-group{ + padding:1rem 0; } @media (min-width: 1200px) { .editor > fx-fore { @@ -912,6 +916,9 @@ input:focus, select:focus, textarea:focus{ #findspot-ctrl{ display:block; } +#findspot-ctrl pb-popover{ + margin-left:2rem; +} #places-list { border-collapse: collapse; width: 100%; diff --git a/src/templates/geo-picker.html b/src/templates/geo-picker.html index 350e975..5abd764 100644 --- a/src/templates/geo-picker.html +++ b/src/templates/geo-picker.html @@ -92,7 +92,7 @@
- + From 84ddba910102ef4cf758d7cbabc4f1596f5e457d Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 2 Oct 2025 19:10:49 +0200 Subject: [PATCH 060/254] restructured and modularized sections into separate files, basic styling --- src/templates/edit.html | 1599 +------------------ src/templates/geo-picker.html | 2 +- src/templates/parts/bibliography.html | 111 ++ src/templates/parts/commentary.html | 69 + src/templates/parts/dating.html | 68 + src/templates/parts/edition.html | 332 ++++ src/templates/parts/editor.html | 23 + src/templates/parts/finddate.html | 45 + src/templates/parts/findspot.html | 31 + src/templates/parts/historic-relevance.html | 88 + src/templates/parts/identifier.html | 22 + src/templates/parts/images.html | 103 ++ src/templates/parts/inscription-field.html | 40 + src/templates/parts/languages.html | 62 + src/templates/parts/location.html | 44 + src/templates/parts/object.html | 22 + src/templates/parts/objectdesc.html | 146 ++ src/templates/parts/pictures.html | 42 + src/templates/parts/prev-editions.html | 97 ++ src/templates/parts/references.html | 21 + src/templates/parts/text-description.html | 69 + src/templates/parts/transmission.html | 102 ++ src/templates/parts/verification.html | 81 + 23 files changed, 1646 insertions(+), 1573 deletions(-) create mode 100644 src/templates/parts/bibliography.html create mode 100644 src/templates/parts/commentary.html create mode 100644 src/templates/parts/dating.html create mode 100644 src/templates/parts/edition.html create mode 100644 src/templates/parts/editor.html create mode 100644 src/templates/parts/finddate.html create mode 100644 src/templates/parts/findspot.html create mode 100644 src/templates/parts/historic-relevance.html create mode 100644 src/templates/parts/identifier.html create mode 100644 src/templates/parts/images.html create mode 100644 src/templates/parts/inscription-field.html create mode 100644 src/templates/parts/languages.html create mode 100644 src/templates/parts/location.html create mode 100644 src/templates/parts/object.html create mode 100644 src/templates/parts/objectdesc.html create mode 100644 src/templates/parts/pictures.html create mode 100644 src/templates/parts/prev-editions.html create mode 100644 src/templates/parts/references.html create mode 100644 src/templates/parts/text-description.html create mode 100644 src/templates/parts/transmission.html create mode 100644 src/templates/parts/verification.html diff --git a/src/templates/edit.html b/src/templates/edit.html index 661c80e..f17d751 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -19,8 +19,7 @@ - + @@ -60,7 +59,7 @@
- + @@ -69,6 +68,7 @@ submission="s-load-inscription" if="exists(instance('params')/id)"> true + - -

- Identifier -

- - - - - - - - - - - + +
- -

Inscription

-
- - Fundort - - - - - - - - - - - - - - -
+ +
-
- - Fundjahr/-datum - - - - - - - - - - - Ungültiges Datum - - - - - - - - Ungültiges Datum - - - - - - - - Ungültiges Datum - - - - - - -
+ +
-
- - - - -
- - - -
- - - - - - -
- - - - - - - - - -
-
+
-
- - Objektbeschreibung - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - -
- - - - - - - - - - -
- - - - - - - - - - -
-
+ +
-
- - Inschriftfeld - - - - - - - - - - - - - - - - - - - - - - -
+
-
- - - - - - - - - - - - -
- - - - - - - - - - - -
- - - - - - - -
+
-
- - Language(s) - - - - - - - - - - - - - - - - -
+
-
- - Verifizierung - - - - - - - - -
- - - - - - - - - - - - - -
-
+

@@ -1039,623 +516,20 @@

-
- - Überlieferung - - - - - - - -
- - - - - - - - - - - - -
-
-
- - Ältere Editionen - - - - - - -
- - - - +
+
+
+
+
- - - - - - - - -
-
-
- - Bibliographie - - - - - - -
- - - - - - - - - - - - - -
-
-
- - Bilder (Fotos, Zeichnungen, Abklatsche) - - - - - - -
- - - - - - - - - - - - - - -
-
-
- - Abbildungen (URLs) - - - - - - -
- - - - - - - - - - - - -
-
+
-
- - Datierung - - - - - - - - - - - - - Ungültiges Datum - - -
- - - - - - Ungültiges Datum - - -
-
- - - - - - Ungültiges Datum - - -
- - - - \ - - - -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
@@ -1663,438 +537,19 @@

- - - - - +
- - - - - - -
- - Textedition - -
-

- Transkription -

- - -
Leiden+ Editor
-
- - - - - - - - - - - - - - - - - -
-
EpiDoc XML Editor
-
- - - - - - - - - - - - - - - - - - -
- - -

- Display -

-
-
-
-

- Apparatus -

- - - -
- - - - - - - - - - - - -
- - -
- - - -
-

- Übersetzung -

- - - -
- - - - - - - - - -
- - - - - - -
-
+
-
- - Kommentar - - - -
- - - - - - - - - - - - -
- - -
+
+
diff --git a/src/templates/geo-picker.html b/src/templates/geo-picker.html index 5abd764..bdc79f0 100644 --- a/src/templates/geo-picker.html +++ b/src/templates/geo-picker.html @@ -14,7 +14,7 @@ - + diff --git a/src/templates/parts/bibliography.html b/src/templates/parts/bibliography.html new file mode 100644 index 0000000..b23b4d1 --- /dev/null +++ b/src/templates/parts/bibliography.html @@ -0,0 +1,111 @@ +
+ + Bibliographie + + + + + + +
+ + + + + + + + + + + + + +
+
diff --git a/src/templates/parts/commentary.html b/src/templates/parts/commentary.html new file mode 100644 index 0000000..65ca1f2 --- /dev/null +++ b/src/templates/parts/commentary.html @@ -0,0 +1,69 @@ +
+ + Kommentar + + + +
+ + + + + + + + + + + + +
+ + +
diff --git a/src/templates/parts/dating.html b/src/templates/parts/dating.html new file mode 100644 index 0000000..d429e84 --- /dev/null +++ b/src/templates/parts/dating.html @@ -0,0 +1,68 @@ +
+ + Datierung + + + + + + + + + + + + + Ungültiges Datum + + +
+ + + + + + Ungültiges Datum + + +
+
+ + + + + + Ungültiges Datum + + +
+ + + + \ + + + +
diff --git a/src/templates/parts/edition.html b/src/templates/parts/edition.html new file mode 100644 index 0000000..1ad0757 --- /dev/null +++ b/src/templates/parts/edition.html @@ -0,0 +1,332 @@ +
+ + Textedition + +
+

+ Transkription +

+ + +
Leiden+ Editor
+
+ + + + + + + + + + + + + + + + + +
+
EpiDoc XML Editor
+
+ + + + + + + + + + + + + + + + + + +
+ + +

+ Display +

+
+
+
+

+ Apparatus +

+ + + +
+ + + + + + + + + + + + +
+ + +
+ + + +
+

+ Übersetzung +

+ + + +
+ + + + + + + + + +
+ + + + + + +
+
diff --git a/src/templates/parts/editor.html b/src/templates/parts/editor.html new file mode 100644 index 0000000..926924f --- /dev/null +++ b/src/templates/parts/editor.html @@ -0,0 +1,23 @@ + + + + + + + + + + + + diff --git a/src/templates/parts/finddate.html b/src/templates/parts/finddate.html new file mode 100644 index 0000000..569fe76 --- /dev/null +++ b/src/templates/parts/finddate.html @@ -0,0 +1,45 @@ +
+ + Fundjahr/-datum + + + + + + + + + + + Ungültiges Datum + + + + + + + + Ungültiges Datum + + + + + + + + Ungültiges Datum + + + + + + +
diff --git a/src/templates/parts/findspot.html b/src/templates/parts/findspot.html new file mode 100644 index 0000000..1120eb5 --- /dev/null +++ b/src/templates/parts/findspot.html @@ -0,0 +1,31 @@ + +
+ + Fundort + + + + + + + + + + + + + + +
diff --git a/src/templates/parts/historic-relevance.html b/src/templates/parts/historic-relevance.html new file mode 100644 index 0000000..ab342bd --- /dev/null +++ b/src/templates/parts/historic-relevance.html @@ -0,0 +1,88 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/src/templates/parts/identifier.html b/src/templates/parts/identifier.html new file mode 100644 index 0000000..8ff1c3e --- /dev/null +++ b/src/templates/parts/identifier.html @@ -0,0 +1,22 @@ + +

+ Identifier +

+ + + + + + + + + + + + + + diff --git a/src/templates/parts/images.html b/src/templates/parts/images.html new file mode 100644 index 0000000..756ba9c --- /dev/null +++ b/src/templates/parts/images.html @@ -0,0 +1,103 @@ +
+ + Bilder (Fotos, Zeichnungen, Abklatsche) + + + + + + +
+ + + + + + + + + + + + + + +
+
diff --git a/src/templates/parts/inscription-field.html b/src/templates/parts/inscription-field.html new file mode 100644 index 0000000..6ed6df4 --- /dev/null +++ b/src/templates/parts/inscription-field.html @@ -0,0 +1,40 @@ +
+ + Inschriftfeld + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/src/templates/parts/languages.html b/src/templates/parts/languages.html new file mode 100644 index 0000000..3226924 --- /dev/null +++ b/src/templates/parts/languages.html @@ -0,0 +1,62 @@ +
+ + Language(s) + + + + + + + + + + + + + + + + +
diff --git a/src/templates/parts/location.html b/src/templates/parts/location.html new file mode 100644 index 0000000..0338b86 --- /dev/null +++ b/src/templates/parts/location.html @@ -0,0 +1,44 @@ +
+ + + + +
+ + + +
+ + + + + + +
+ + + + + + + + + +
+
diff --git a/src/templates/parts/object.html b/src/templates/parts/object.html new file mode 100644 index 0000000..8ff1c3e --- /dev/null +++ b/src/templates/parts/object.html @@ -0,0 +1,22 @@ + +

+ Identifier +

+ + + + + + + + + + + + + + diff --git a/src/templates/parts/objectdesc.html b/src/templates/parts/objectdesc.html new file mode 100644 index 0000000..333cad4 --- /dev/null +++ b/src/templates/parts/objectdesc.html @@ -0,0 +1,146 @@ +
+ + Objektbeschreibung + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ + + + + + + + + + +
+ + + + + + + + + + +
+
diff --git a/src/templates/parts/pictures.html b/src/templates/parts/pictures.html new file mode 100644 index 0000000..b6e0cdb --- /dev/null +++ b/src/templates/parts/pictures.html @@ -0,0 +1,42 @@ +
+ + Abbildungen (URLs) + + + + + + +
+ + + + + + + + + + + + +
+
diff --git a/src/templates/parts/prev-editions.html b/src/templates/parts/prev-editions.html new file mode 100644 index 0000000..2af1b98 --- /dev/null +++ b/src/templates/parts/prev-editions.html @@ -0,0 +1,97 @@ +
+ + Ältere Editionen + + + + + + +
+ + + + + + + + + + + + + +
+
diff --git a/src/templates/parts/references.html b/src/templates/parts/references.html new file mode 100644 index 0000000..5585965 --- /dev/null +++ b/src/templates/parts/references.html @@ -0,0 +1,21 @@ + +

+ Referenzen + + + + +

+ +
+ +
+
+ + +
+
+ + diff --git a/src/templates/parts/text-description.html b/src/templates/parts/text-description.html new file mode 100644 index 0000000..2be98c5 --- /dev/null +++ b/src/templates/parts/text-description.html @@ -0,0 +1,69 @@ +
+ + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + + +
diff --git a/src/templates/parts/transmission.html b/src/templates/parts/transmission.html new file mode 100644 index 0000000..f0d389d --- /dev/null +++ b/src/templates/parts/transmission.html @@ -0,0 +1,102 @@ +
+ + Überlieferung + + + + + + + +
+ + + + + + + + + + + + +
+
diff --git a/src/templates/parts/verification.html b/src/templates/parts/verification.html new file mode 100644 index 0000000..075eda5 --- /dev/null +++ b/src/templates/parts/verification.html @@ -0,0 +1,81 @@ +
+ + Verifizierung + + + + + + + + +
+ + + + + + + + + + + + + +
+
From f222a3b2e59e0a6f72bdc77835ef5c09f7b449da Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Mon, 6 Oct 2025 19:09:11 +0200 Subject: [PATCH 061/254] a draft API for Zotero catching the use cases from current form plus basic syncing (not activated in any form) --- src/modules/lib/zotero-api.json | 148 ++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 src/modules/lib/zotero-api.json diff --git a/src/modules/lib/zotero-api.json b/src/modules/lib/zotero-api.json new file mode 100644 index 0000000..c570370 --- /dev/null +++ b/src/modules/lib/zotero-api.json @@ -0,0 +1,148 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Zotero Cache API (Group 2519759)", + "version": "0.3.0", + "description": "Local cache API for Zotero group 2519759. JSON-only: search, read, create, and tags." + }, + "servers": [{ "url": "/api" }], + "paths": { + "/z/sync": { + "post": { + "summary": "Incrementally sync local cache.", + "responses": { + "200": { + "description": "Sync result.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { "type": "string", "example": "ok" }, + "updated": { "type": "integer" }, + "libraryVersion": { "type": "integer" } + } + } + } + } + }, + "304": { "description": "No updates." } + } + } + }, + + "/z/items/search": { + "get": { + "summary": "Search cached items. JSON only.", + "parameters": [ + { "in": "query", "name": "q", "schema": { "type": "string" }, "description": "Full-text over title/creators/DOI." }, + { "in": "query", "name": "tag", "schema": { "type": "string" }, "description": "Restrict to items having this tag." }, + { "in": "query", "name": "limit", "schema": { "type": "integer", "default": 15 } } + ], + "responses": { + "200": { + "description": "Array of items (key + data).", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "required": ["key", "data"], + "properties": { + "key": { "type": "string" }, + "data": { "type": "object", "description": "Zotero item 'data' JSON (stored pristine)." } + } + } + } + } + } + } + } + } + }, + + "/z/items/{key}": { + "get": { + "summary": "Get a cached item by Zotero key. JSON only.", + "parameters": [ + { "in": "path", "name": "key", "required": true, "schema": { "type": "string" } } + ], + "responses": { + "200": { + "description": "Item JSON.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["key", "data"], + "properties": { + "key": { "type": "string" }, + "data": { "type": "object" } + } + } + } + } + }, + "404": { "description": "Not found." } + } + } + }, + + "/z/items": { + "post": { + "summary": "Create a new Zotero item and update cache (JSON-only).", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["item", "tags"], + "properties": { + "item": { "type": "object", "description": "Zotero item 'data' JSON." }, + "tags": { "type": "array", "items": { "type": "string" } } + } + } + } + } + }, + "responses": { + "201": { + "description": "Created.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { "type": "string", "example": "created" }, + "key": { "type": "string" } + } + } + } + } + }, + "409": { "description": "Library locked / conflict." }, + "412": { "description": "Precondition failed (version mismatch)." }, + "400": { "description": "Invalid item payload." } + } + } + }, + + "/z/tags": { + "get": { + "summary": "List distinct tags present in the cached group (for pickers).", + "responses": { + "200": { + "description": "Array of tags.", + "content": { + "application/json": { + "schema": { "type": "array", "items": { "type": "string" } } + } + } + } + } + } + } + } +} From 80995a4af6fce5a6df1df135ba5f5ad52adf2b95 Mon Sep 17 00:00:00 2001 From: Magdalena Turska Date: Tue, 7 Oct 2025 13:36:32 +0200 Subject: [PATCH 062/254] fix: api endpoint --- src/modules/lib/api.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/lib/api.json b/src/modules/lib/api.json index db5baf2..879d9b3 100644 --- a/src/modules/lib/api.json +++ b/src/modules/lib/api.json @@ -8,7 +8,7 @@ "servers": [ { "description": "Endpoint for testing on localhost", - "url": "/exist/apps/tei-publisher" + "url": "/exist/apps/edep" } ], "tags": [ From af72f030112a003803614835c13a32c1ccaea999 Mon Sep 17 00:00:00 2001 From: Martin Middel Date: Tue, 7 Oct 2025 13:46:10 +0200 Subject: [PATCH 063/254] fix(places): make place form part work with country.xml --- src/modules/custom-api.xql | 116 ++++++++++++++++----------------- src/templates/edit.html | 4 +- src/templates/fore/places.html | 10 ++- src/templates/geo-picker.html | 9 ++- 4 files changed, 75 insertions(+), 64 deletions(-) diff --git a/src/modules/custom-api.xql b/src/modules/custom-api.xql index c5bd2d9..c9a9f32 100644 --- a/src/modules/custom-api.xql +++ b/src/modules/custom-api.xql @@ -39,7 +39,7 @@ declare function api:writing($request as map(*)) { } }; -declare function api:typeins($request as map(*)) { +declare function api:typeins($request as map(*)) { try { {doc($config:data-root || "/typeins.xml")} } catch * { @@ -65,7 +65,7 @@ declare function api:objtyp($request as map(*)) { declare function api:decor($request as map(*)) { try { - {doc($config:data-root || "/decor.xml")/items/item} + {doc($config:data-root || "/decor.xml")/items/item} } catch * { () } @@ -89,13 +89,13 @@ declare function api:places-browse($request as map(*)) { collection($config:data-root || "/places")//tei:place[contains(@xml:id, $search)] else collection($config:data-root || "/places")//tei:place - let $sorted := + let $sorted := for $place in $places order by $place/tei:placeName[@type="modern"] return $place - let $letter := - if (count($places) < $limit) then + let $letter := + if (count($places) < $limit) then "Alle" else if ($letterParam = '') then substring($sorted[1], 1, 1) => upper-case() @@ -165,10 +165,10 @@ declare function api:find-spot($request as map(*)) { let $placeIds := $xml//tei:origPlace/@corresp let $places := for $placeId in $placeIds return collection($config:data-root || "/places")/id($placeId) return - array { - for $place in $places + array { + for $place in $places let $tokenized := tokenize($place/tei:location/tei:geo, ',\s*') - return + return map { "latitude":$tokenized[1], "longitude":$tokenized[2], @@ -177,13 +177,13 @@ declare function api:find-spot($request as map(*)) { } }; -declare function api:load-place($request as map(*)) { - let $return := doc(concat($config:places, $request?parameters?id, ".xml")) - return try { - $return - } catch * { - () - } +declare function api:load-place ($request as map(*)) { + let $loc := concat($config:places, $request?parameters?id, ".xml") + return if (not(doc-available($loc))) then + error($errors:NOT_FOUND) + else + let $return := doc($loc) + return try { $return } catch * { () } }; declare function api:geopicker-places($request as map(*)) { @@ -217,7 +217,7 @@ declare function api:places-add($request as map(*)) { let $store := xmldb:store($config:places, concat("G", $id-new, ".xml"), $request?body) let $update := update insert attribute xml:id {concat("G", $id-new)} into doc(concat($config:places, "G", $id-new, ".xml"))/tei:place return concat("G",$id-new) - + return try { doc(concat($config:places, $id, ".xml")) @@ -235,13 +235,13 @@ declare function api:people-browse($request as map(*)) { collection($config:data-root || "/people")//tei:person[ft:query(tei:persName, $search || '*')] else collection($config:data-root || "/people")//tei:person - let $sorted := + let $sorted := for $person in $people order by $person/tei:persName[@type='nomen'] return $person - let $letter := - if (count($people) < $limit) then + let $letter := + if (count($people) < $limit) then "Alle" else if ($letterParam = '') then substring($sorted[1], 1, 1) => upper-case() @@ -298,12 +298,12 @@ declare function api:output-person($list, $category as xs:string, $search as xs: }; declare function api:load-person($request as map(*)) { - let $return := doc(concat($config:people, $request?parameters?id, ".xml")) - return try { - $return - } catch * { - () - } + let $loc := concat($config:people, $request?parameters?id, ".xml") + return if (not(doc-available($loc))) then + error($errors:NOT_FOUND) + else + let $return := doc($loc) + return try { $return } catch * { () } }; declare function api:person-add($request as map(*)) { @@ -320,14 +320,14 @@ declare function api:person-add($request as map(*)) { let $id-new := if (empty($ids)) then "000000" else format-number(xs:integer(replace($ids[last()], "P", "")) + 1, "000000") let $withId := - { + { $request?body//tei:person/@sex, - $request?body/tei:person/* + $request?body/tei:person/* } let $store := xmldb:store($config:people, concat("P", $id-new, ".xml"), $withId) return concat("P",$id-new) - + return try { doc(concat($config:people, $id, ".xml")) @@ -337,13 +337,13 @@ declare function api:person-add($request as map(*)) { }; declare function api:inscription($request as map(*)) { - let $check-collection := - if(not(xmldb:collection-available($config:inscription))) then - xmldb:create-collection("/", $config:inscription) - else + let $check-collection := + if(not(xmldb:collection-available($config:inscription))) then + xmldb:create-collection("/", $config:inscription) + else () let $collection := $config:data-root || "/" || $request?parameters?collection - let $id := + let $id := if ($request?parameters?id and $request?parameters?id != '') then let $store := xmldb:store($collection, concat($request?parameters?id, ".xml"), api:clean($request?body, (), true())) return $request?body//tei:idno[@type="EDEp"]/text() @@ -384,7 +384,7 @@ declare function api:inscription-template($request as map(*)) { $merged else doc($config:inscription-templ) - + return try { $doc } catch * { @@ -438,9 +438,9 @@ declare %private function api:postprocess($nodes as node()*, $edepId as xs:strin element { node-name($node) } { $node/@*, $node/tei:change[@type='created'], - } case element() return @@ -469,7 +469,7 @@ declare function api:clean-namespace($nodes as node()*) { declare function api:render($request as map(*)) { let $type := $request?parameters?type - let $xml := + let $xml := switch ($type) case "transcription" return $request?body//tei:div[@type="edition"] @@ -496,7 +496,7 @@ in the template :) declare %private function api:find-counterpart($nodeTemplate as element(), $input as node()) as item()* { (: List of candidates is created based on the name of the element and its ancestors. In addition we look for the values of the attribute @type to disambiguate from - and for the values of @scheme to disambiguate the elements + and for the values of @scheme to disambiguate the elements When working on the main template, we look at all the ancestors, if we are in a secondary template (see condition) we only check the parent :) let $candidates := if ($nodeTemplate/ancestor-or-self::tei:TEI) then $input/descendant::*[local-name() eq $nodeTemplate/local-name()] @@ -507,13 +507,13 @@ declare %private function api:find-counterpart($nodeTemplate as element(), $inpu [every $scheme in $nodeTemplate/ancestor-or-self::*[@scheme ne '']/@scheme satisfies $scheme = ./ancestor-or-self::*/@scheme] [if (@corresp = ./root()/descendant::tei:msPart[@type eq 'fragment']) then false() else true()] - else + else $input/descendant::*[local-name() eq $nodeTemplate/local-name()] [parent::*/local-name() eq $nodeTemplate/parent::*/local-name()] [if ($nodeTemplate[@type and (@type ne '')]) then self::*[@type eq $nodeTemplate/@type] else true()] (: If the candidates are siblings, we also selected the first one. If at this point we have more than one candidate, throw an error with the element name :) - let $counterpart := + let $counterpart := if (count($candidates/parent::*) eq 1) then $candidates[1] else if (count($candidates) <= 1) then @@ -527,7 +527,7 @@ declare %private function api:find-counterpart($nodeTemplate as element(), $inpu declare %private function api:process-children($nodeTemplate as element(), $nodeInput as element()) as item()* { (: if the node from the input file only contais a text node, or mixed content, then get its children :) if ($nodeInput[((count(child::node()) eq 1) and (text()[string-length(replace(., '\s+', '')) ne 0])) or - ((text()[string-length(replace(., '\s+', '')) ne 0]) and child::element())]) + ((text()[string-length(replace(., '\s+', '')) ne 0]) and child::element())]) then api:complete-input($nodeInput/node()) else @@ -543,20 +543,20 @@ declare %private function api:process-children($nodeTemplate as element(), $node (:function to complete the input with elements that are in an secondary template :) declare %private function api:complete-input($nodes as node()*) as node()* { - for $node in $nodes - return + for $node in $nodes + return typeswitch($node) case element(tei:bibl) return let $templateBibl := (doc('/db/apps/edep/templates/fore/templates.xml')//tei:bibl)[1] - return + return { ($node/@type, $templateBibl/@type)[1] } {($node/node(), $templateBibl/*[not(local-name() = $node/node()/local-name())])} - default + default return $node }; - + (:function to add @corresp attribute values when elements are copied from the template :) declare %private function api:add-corresp($nodeTemplate as element(), $input as node()) as element()* { if ($nodeTemplate[@corresp]) @@ -564,28 +564,28 @@ declare %private function api:add-corresp($nodeTemplate as element(), $input as for $id in $input/root()/descendant::tei:msPart/@xml:id let $correspVal:= '#' || $id let $att := attribute {'corresp'} {$correspVal} - return + return element {QName("http://www.tei-c.org/ns/1.0", $nodeTemplate/local-name())} { $nodeTemplate/@*[not(name() eq 'corresp')] | $att, $nodeTemplate/node() } -else +else $nodeTemplate }; - + declare %private function api:compare-elements($nodeTemplate as element(), $nodeInput as element()) as element(){ if (deep-equal($nodeTemplate, $nodeInput)) then $nodeTemplate else (: if the number of attributes is not the same, get the missing attributes from the template:) if (count($nodeInput/@*) ne count($nodeTemplate/@*)) - then + then let $emptyAttsNames := for $att in $nodeTemplate/@* return $att[not(name() = $nodeInput/@*/name())]/name() let $emptyAtts := for $attName in $emptyAttsNames return - attribute {$attName} {""} + attribute {$attName} {""} return (: we return an element with all the attributes and then we process its contents :) element {QName("http://www.tei-c.org/ns/1.0", $nodeTemplate/local-name())} @@ -608,10 +608,10 @@ declare %private function api:reconstruct-tree($tmplNodes as element()*, $input let $name := $tmpl/local-name() let $counterpart := api:find-counterpart($tmpl, $input) return - (: if we find an equivalent element, we return more than one item: on one hand, + (: if we find an equivalent element, we return more than one item: on one hand, the result of processing the “counterpart” element, on the other, additional operations are done to handle repeateable elements :) - if ($counterpart) then + if ($counterpart) then (api:compare-elements($tmpl, $counterpart), (: create as many div elements as necessary attending to the @corresp attributes :) (: if ($counterpart[@corresp][local-name() = ('div')]) :) @@ -637,16 +637,16 @@ declare %private function api:reconstruct-tree($tmplNodes as element()*, $input if ($counterpart[@type eq 'main']/following-sibling::*[1][self::tei:msPart[@type eq 'fragment']]) then api:complete-input($counterpart/following-sibling::tei:msPart[@type eq 'fragment']) else - + (: Second scenario: there are elements in the input file, not present in the template. For those cases we look in the element in the input file being processed has a following-sibling that it’s not present in the template.
elements are excluded to avoid the duplication of div[@type eq 'textpart'] of fragments:) if ($counterpart[not(local-name() eq 'div')] and not($counterpart/following-sibling::*[local-name() = $tmpl/following-sibling::*/local-name()])) then $counterpart/following-sibling::*[not(local-name() = $tmpl/following-sibling::*/local-name())] else () - - - ) + + + ) else typeswitch($tmpl) case element(tei:div) return api:add-corresp($tmpl, $input) diff --git a/src/templates/edit.html b/src/templates/edit.html index f17d751..e61a4db 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -122,7 +122,7 @@ - + hidden false @@ -344,6 +344,8 @@ + +
- + diff --git a/src/templates/parts/images.html b/src/templates/parts/images.html index 756ba9c..8210df2 100644 --- a/src/templates/parts/images.html +++ b/src/templates/parts/images.html @@ -27,9 +27,6 @@ context="listBibl[@type='images']" ref="bibl" origin="instance('templates')//listBibl[@type='images']/bibl"> - @@ -47,12 +44,11 @@ >
- - +
@@ -64,7 +60,7 @@

- + diff --git a/src/templates/parts/prev-editions.html b/src/templates/parts/prev-editions.html index 53fc03e..9de3c34 100644 --- a/src/templates/parts/prev-editions.html +++ b/src/templates/parts/prev-editions.html @@ -21,9 +21,6 @@ context="listBibl[@type='previousEditions']" ref="bibl" origin="instance('templates')//bibl"> - diff --git a/src/templates/parts/transmission.html b/src/templates/parts/transmission.html index f0d389d..f3e3e60 100644 --- a/src/templates/parts/transmission.html +++ b/src/templates/parts/transmission.html @@ -23,9 +23,6 @@ context="listBibl[@type='transmission']" ref="bibl" origin="instance('templates')//bibl"> - @@ -45,12 +42,11 @@ >
- - +
@@ -62,7 +58,7 @@

- + From ee9c548128fd8300ed9adffd169341e9f6a41cd2 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Wed, 8 Oct 2025 14:54:28 +0200 Subject: [PATCH 073/254] a draft API for Zotero catching the use cases from current form plus basic syncing (not activated in any form) --- src/templates/edit.html | 132 +++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 30 deletions(-) diff --git a/src/templates/edit.html b/src/templates/edit.html index e61a4db..3d79527 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -19,11 +19,10 @@ - + - @@ -68,7 +67,6 @@ submission="s-load-inscription" if="exists(instance('params')/id)"> true - -
- - - +
+ +

Inscription

-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+

@@ -518,20 +554,46 @@

-
-
-
-
-
- +
+
+
+
+
-
- - -
+
+ +
@@ -539,19 +601,29 @@

-
- +
-
+
-
- +
From 78c803841aaafd325321cb145b28b2553230d9f7 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 12:52:02 +0200 Subject: [PATCH 074/254] added zotero config parameters --- src/modules/config.xqm | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/modules/config.xqm b/src/modules/config.xqm index f51f4b4..cd78575 100644 --- a/src/modules/config.xqm +++ b/src/modules/config.xqm @@ -19,11 +19,11 @@ declare namespace tei="http://www.tei-c.org/ns/1.0"; (:~ : Define where places are located - :) declare variable $config:places := $config:data-root || "/places/"; declare variable $config:people := $config:data-root || "/people/"; declare variable $config:inscription := $config:data-root || "/workspace/"; declare variable $config:inscription-templ := $config:app-root || "/templates/fore/epidoc-template.xml"; + :) (:~~ : The version of the pb-components webcomponents library to be used by this app. @@ -321,6 +321,33 @@ declare variable $config:context-path := :) declare variable $config:data-root := repo:get-root() || "edep-data"; +(:~ + : Define where places are located + :) + declare variable $config:places := $config:data-root || "/places/"; + declare variable $config:people := $config:data-root || "/people/"; + declare variable $config:inscription := $config:data-root || "/workspace/"; + declare variable $config:inscription-templ := $config:app-root || "/templates/fore/epidoc-template.xml"; + +(: ZOTERO CONFIG :) +(: Base URL of Zotero Web API :) +declare variable $config:zotero-api-base := "https://api.zotero.org"; + +(: Optional API key; leave empty for public groups :) +declare variable $config:zotero-api-key := ""; + +(: Your group id :) +declare variable $config:zotero-group-id := "2529759"; + +(: Base dir where all groups live; must already exist :) +declare variable $config:zotero-base-dir := $config:data-root || "/zotero/groups"; + +(: Derived paths for this group :) +declare variable $config:zotero-group-dir := $config:zotero-base-dir || "/" || $config:zotero-group-id; +declare variable $config:zotero-items-dir := $config:zotero-group-dir || "/items"; +declare variable $config:zotero-meta-path := $config:zotero-group-dir || "/meta.json"; + + (:~ : The root of the collection hierarchy whose files should be displayed : on the entry page. Can be different from $config:data-root. From 36aa689a145bce8a888dd3fd7cb2bd764dc78592 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 12:58:50 +0200 Subject: [PATCH 075/254] creating zotero cache collections in post-install (edep-data needs to be there when edep is installed) --- doc/zotero-cache.md | 187 +++++++++++++++++++++++++++++++++++++++++++ src/post-install.xql | 85 ++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 doc/zotero-cache.md diff --git a/doc/zotero-cache.md b/doc/zotero-cache.md new file mode 100644 index 0000000..ad2b4b2 --- /dev/null +++ b/doc/zotero-cache.md @@ -0,0 +1,187 @@ +# Zotero Group Cache — Architecture & API (eXist-db 6.4) + +This document describes the local Zotero cache used by the app, the expected collection layout in eXist‑db, the config variables, the sync endpoint, and the installation steps. + +> Target stack: **eXist‑db 6.4.x**, **Roaster** (OpenAPI router), **Fore** frontend. +> Zotero library: **Group** (public or private). API key is **optional** for public groups. + +--- + +## 1) Collection layout + +Only the **group** collection is created dynamically by the post‑install script. The **base** and **items** collections are expected to exist after installation. + +``` +/db +└── zotero-cache/ (collection) [pre-created] + └── groups/ (collection) [pre-created] + └── / (collection) [created by post-install] + ├── items/ (collection) [created by post-install] + │ ├── ABC12345.json (application/json) ← Zotero item “data” for key ABC12345 + │ ├── 9KLMNO67.json (application/json) + │ └── ... (application/json) + └── meta.json (application/json) ← created/seeded by post-install +``` + +### `meta.json` (seeded by post-install) +```json +{ + "libraryVersion": 0, + "syncedAt": "YYYY-MM-DDThh:mm:ssZ" +} +``` +- **libraryVersion**: last known Zotero `Last-Modified-Version` for the group (int). +- **syncedAt**: timestamp of last local sync. + +### `items/.json` +One file per Zotero item **key**. Stored content is the **pristine Zotero `data` object** (no wrapping). Example: +```json +{ + "itemType": "journalArticle", + "title": "Example Title", + "creators": [{ "creatorType": "author", "firstName": "Jane", "lastName": "Doe" }], + "date": "2020", + "DOI": "10.1234/example.doi", + "url": "https://example.org", + "tags": [{ "tag": "project:demo" }, { "tag": "public" }] +} +``` + +--- + +## 2) Config module (`config.xqm`) + +The application reads **config variables** directly (hyphenated names), and also provides **function wrappers** for legacy callers. Keep both to avoid router/package regressions. + +```xquery +xquery version "3.1"; +module namespace config = "http://example.org/config"; + +(: ── variables (used directly) ── :) +declare variable $config:zotero-api-base as xs:string := "https://api.zotero.org"; +declare variable $config:zotero-api-key as xs:string := ""; (: optional for public :) +declare variable $config:zotero-group-id as xs:integer := 2529759; + +declare variable $config:zotero-base-dir as xs:string := "/db/zotero-cache/groups"; +declare variable $config:zotero-group-dir as xs:string := concat($config:zotero-base-dir, "/", $config:zotero-group-id); +declare variable $config:zotero-items-dir as xs:string := concat($config:zotero-group-dir, "/items"); +declare variable $config:zotero-meta-path as xs:string := concat($config:zotero-group-dir, "/meta.json"); +``` + +**Notes** +- For **public** Zotero groups, `$config:zotero-api-key` may be the empty string; the sync omits the `Authorization` header in that case. +- If you change the **group id**, post‑install must be run again to create the new group layout. + +--- + +## 3) Installation bootstrap (`post-install.xql`) + +The post‑install script should: +1. Create collections: `$config:zotero-base-dir`, `$config:zotero-group-dir`, `$config:zotero-items-dir` (stepwise under `/db`). +2. Seed `$config:zotero-meta-path` with the template JSON (media type `application/json`) if missing. + +A minimal post‑install does: +- `local:mkcol()` to ensure collections (stepwise, eXist‑6.4 safe) +- seeds `meta.json` via 4‑arg `xmldb:store(..., \"application/json\")` +- returns a JSON summary + +> After installation, the **sync** endpoint will not create collections or seed meta; it assumes post‑install of edep app! prepared the layout. + +--- + +## 4) Sync endpoint + +**Route**: `POST /api/z/sync` → `zotero:sync` +**Behavior**: Incremental, paginated sync from Zotero group into local cache. + +### Request → Zotero +- `GET {zotero-api-base}/groups/{zotero-group-id}/items?since={libraryVersion}&limit=100` +- Headers: + - `Zotero-API-Version: 3` + - `Authorization: Bearer {apiKey}` (only if `$config:zotero-api-key` is non-empty) + - `If-Modified-Since-Version: {libraryVersion}` (only when `libraryVersion > 0`) + +### Response (local API) +```json +{ "status": "ok", "updated": , "libraryVersion": } +``` +- `status`: `"ok"` or `"error"` +- `updated`: number of items locally stored across all pages +- `libraryVersion`: Zotero’s `Last-Modified-Version` captured and saved to `meta.json` + +### Pagination +- Follows the `Link` response header (`rel="next"`) until exhausted. + +### Error handling +When Zotero replies with a non‑200 status: +```json +{ "status":"error", "httpStatus": , "backoff":"...", "retryAfter":"..." } +``` +If Zotero returns `304 Not Modified`, the local API responds with: +```json +{ "status":"ok", "updated": 0, "libraryVersion": } +``` + +--- + +## 5) Router wiring (Roaster) + +Example OpenAPI snippet (YAML): +```yaml +paths: + /api/z/sync: + post: + summary: Sync local cache with Zotero group + x-roaster-xquery: + module: /db/apps/edep/modules/lib/zotero.xql + function: zotero:sync # supports sync#2 and sync#1 (wrapper) + responses: + '200': + description: Sync result +``` + +Handler arities to be export-safe: +```xquery +(: wrapper for routers expecting sync#1 :) +declare function zotero:sync($config as map(*)) as xs:string { + zotero:sync($config, ) +}; + +(: main handler :) +declare function zotero:sync($config as map(*), $root as element()) as xs:string { + (: run sync and return JSON string :) +}; +``` + +--- + +## 6) Troubleshooting + +- **make sure edep-data.xar has been installed before edep.xar** + post-install of edep.xar creates the needed collections in edep-data + +- **First run fails reading meta** + Ensure post‑install created `meta.json`. If absent, running post‑install again will seed it. + +- **No items written** + Confirm that `$config:zotero-items-dir` exists and is writable. The sync does not create it. + +- **Public group sync** + Leave `$config:zotero-api-key` empty. The Authorization header will be omitted. + +--- + +## 7) Trigger sync (curl) +```bash +curl -X POST 'http://localhost:8080/api/z/sync' +``` + +You should see JSON like: +```json +{ "status":"ok", "updated": 123, "libraryVersion": 4567 } +``` + +--- + +*Document version:* 1.0 +*Last updated:* generated for the current build. diff --git a/src/post-install.xql b/src/post-install.xql index 000b4b5..6dcb08b 100644 --- a/src/post-install.xql +++ b/src/post-install.xql @@ -93,6 +93,90 @@ declare function local:generate-code($collection as xs:string) { ) }; +(:───────────────────────────────────────────────────────────── + : ZOTERO LAYOUT (uses local:mkcol-recursive($collection,$components)) + : append to post-install.xql — no existing code removed + :─────────────────────────────────────────────────────────────:) + +(: split absolute /db path into components after '/db/' :) +declare function local:path-components-after-db($abs as xs:string) as xs:string* { + let $norm := replace($abs, '/+$', '') + let $rel := substring-after($norm, '/db/') + return if ($rel = '' or $rel = $norm) then () else tokenize($rel, '/') +}; + +(: convenience wrapper: create an absolute /db path with mkcol-recursive :) +declare function local:mkcol-abs($abs as xs:string) as empty-sequence() { + let $comps := local:path-components-after-db($abs) + return if (empty($comps)) then () else local:mkcol-recursive('/db', $comps) +}; + +(: 6.4-safe resource existence :) +declare function local:zotero-resource-exists($coll as xs:string, $name as xs:string) as xs:boolean { + if (not(xmldb:collection-available($coll))) then false() + else some $r in xmldb:get-child-resources($coll) satisfies ($r = $name) +}; + +(: split absolute db path → {coll, name} :) +declare function local:zotero-path-split($abs as xs:string) as map(*) { + let $name := tokenize($abs, '/')[last()] + let $coll := substring($abs, 1, string-length($abs) - string-length($name) - 1) + return map{ "coll": $coll, "name": $name } +}; + +(: seed meta.json if missing; returns true() if written :) +declare function local:zotero-seed-meta-if-missing($abs as xs:string) as xs:boolean { + let $ps := local:zotero-path-split($abs) + return + if (not(xmldb:collection-available($ps?coll))) then false() + else if (local:zotero-resource-exists($ps?coll, $ps?name)) then false() + else + try { + let $_ := xmldb:store( + $ps?coll, $ps?name, + serialize( + map{ "libraryVersion": 0, "syncedAt": current-dateTime() }, + map{ "method":"json", "indent": true() } + ), + "application/json" + ) + return true() + } catch * { + false() + } +}; + +(: PUBLIC: ensure base, group, items; then seed meta :) +declare function local:zotero-ensure-layout() as map(*) { + let $_b := local:mkcol-abs($config:zotero-base-dir) + let $_g := local:mkcol-abs($config:zotero-group-dir) + let $_i := local:mkcol-abs($config:zotero-items-dir) + + let $metaSeeded := local:zotero-seed-meta-if-missing($config:zotero-meta-path) + + return map{ + "status": "ok", + "ensured": map{ + "base": xmldb:collection-available($config:zotero-base-dir), + "group": xmldb:collection-available($config:zotero-group-dir), + "items": xmldb:collection-available($config:zotero-items-dir), + "metaSeeded": $metaSeeded + }, + "paths": map{ + "base": $config:zotero-base-dir, + "group": $config:zotero-group-dir, + "items": $config:zotero-items-dir, + "meta": $config:zotero-meta-path + } + } +}; + +(: OPTIONAL JSON summary for logs :) +declare function local:zotero-ensure-layout-json() as xs:string { + serialize(local:zotero-ensure-layout(), map{ "method":"json", "indent": true() }) +}; + + (: API needs dba rights for LaTeX :) sm:chgrp(xs:anyURI($target || "/modules/lib/api-dba.xql"), "dba"), sm:chmod(xs:anyURI($target || "/modules/lib/api-dba.xql"), "rwxr-Sr-x"), @@ -100,6 +184,7 @@ sm:chmod(xs:anyURI($target || "/modules/lib/api-dba.xql"), "rwxr-Sr-x"), local:mkcol($target, "transform"), local:generate-code($target), local:create-data-collection(), +local:zotero-ensure-layout(), let $pmuConfig := pmc:generate-pm-config(($config:odd-available, $config:odd-internal), $config:default-odd, $config:odd-root) return xmldb:store($config:app-root || "/modules", "pm-config.xql", $pmuConfig, "application/xquery") \ No newline at end of file From de8cfb654a1f73a172fb938f412957ae99541157 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 17:09:17 +0200 Subject: [PATCH 076/254] fixing permissions for meta.json file --- src/post-install.xql | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/post-install.xql b/src/post-install.xql index 6dcb08b..f7a571d 100644 --- a/src/post-install.xql +++ b/src/post-install.xql @@ -125,25 +125,41 @@ declare function local:zotero-path-split($abs as xs:string) as map(*) { }; (: seed meta.json if missing; returns true() if written :) +(: seed meta.json if missing, then align its permissions to the parent collection :) declare function local:zotero-seed-meta-if-missing($abs as xs:string) as xs:boolean { let $ps := local:zotero-path-split($abs) return if (not(xmldb:collection-available($ps?coll))) then false() - else if (local:zotero-resource-exists($ps?coll, $ps?name)) then false() else - try { - let $_ := xmldb:store( - $ps?coll, $ps?name, - serialize( - map{ "libraryVersion": 0, "syncedAt": current-dateTime() }, - map{ "method":"json", "indent": true() } - ), - "application/json" - ) - return true() - } catch * { - false() - } + let $exists := local:zotero-resource-exists($ps?coll, $ps?name) + let $created := + if ($exists) then false() + else + try { + let $_ := xmldb:store( + $ps?coll, $ps?name, + serialize( + map { "libraryVersion": 0, "syncedAt": current-dateTime() }, + map { "method":"json", "indent": true() } + ), + "application/json" + ) + return true() + } catch * { false() } + (: ALWAYS try to align perms (whether created just now or already existed) :) + let $_fixPerms := + try { + let $resPath := concat($ps?coll, "/", $ps?name) + let $p := sm:get-permissions($ps?coll) + let $owner := string(($p/@owner, "guest")[1]) + let $group := string(($p/@group, "guest")[1]) + let $mode := string(($p/@mode, "rw-rw-r--")[1]) + let $_1 := sm:chown($resPath, "edep") + let $_2 := sm:chgrp($resPath, "tei") + let $_3 := sm:chmod($resPath, $mode) + return () + } catch * { () } + return $created }; (: PUBLIC: ensure base, group, items; then seed meta :) From e146845d90e6654eee6eca356f7fd12758ae38a4 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 17:09:45 +0200 Subject: [PATCH 077/254] fix the group number --- src/modules/config.xqm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/config.xqm b/src/modules/config.xqm index cd78575..33a0c1b 100644 --- a/src/modules/config.xqm +++ b/src/modules/config.xqm @@ -337,7 +337,7 @@ declare variable $config:zotero-api-base := "https://api.zotero.org"; declare variable $config:zotero-api-key := ""; (: Your group id :) -declare variable $config:zotero-group-id := "2529759"; +declare variable $config:zotero-group-id := "2519759"; (: Base dir where all groups live; must already exist :) declare variable $config:zotero-base-dir := $config:data-root || "/zotero/groups"; From deba36ccba3e2b7fa071b9fc351648b9a08b6d70 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 17:40:20 +0200 Subject: [PATCH 078/254] wip: but basic sync running, problems with updating meta.json --- src/modules/lib/api/zotero.xql | 288 +++++++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 src/modules/lib/api/zotero.xql diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql new file mode 100644 index 0000000..156587a --- /dev/null +++ b/src/modules/lib/api/zotero.xql @@ -0,0 +1,288 @@ +xquery version "3.1"; + +module namespace zotero = "http://teipublisher.com/api/zotero"; + +declare namespace map = "http://www.w3.org/2005/xpath-functions/map"; +declare namespace http = "http://expath.org/ns/http-client"; +declare namespace xmldb = "http://exist-db.org/xquery/xmldb"; +declare namespace util = "http://exist-db.org/xquery/util"; +import module namespace response = "http://exist-db.org/xquery/response"; +(: Import your config module — update the path :) +import module namespace config = "http://www.tei-c.org/tei-simple/config" + at "../../config.xqm"; + +(: ───────────────────────────────────────────────────────── + Helpers + ───────────────────────────────────────────────────────── :) + +declare %private function zotero:json($v as item()*) as xs:string { + serialize($v, map{"method":"json","indent":true()}) +}; + +(: Build headers; Authorization omitted if key is empty :) +(: +declare %private function zotero:headers($extra as element(http:header)*) as element(http:header)* { + let $apiVer := + let $accept := + let $ua := + let $auth := + if (normalize-space($config:zotero-api-key) ne "") then ( + , + + ) else () + return ($apiVer, $accept, $ua, $auth, $extra) +}; +:) + +(: grab header values case-insensitively :) +declare %private function zotero:header($resp as element(http:response), $name as xs:string) as xs:string* { + $resp/http:header[lower-case(@name) = lower-case($name)]/@value/string() +}; + +(: replace your zotero:headers :) +declare %private function zotero:headers($extra as element(http:header)*) as element(http:header)* { + let $apiVer := + let $accept := + let $ua := + let $auth := + if (normalize-space($config:zotero-api-key) ne "") then ( + , + + ) else () + return ($apiVer, $accept, $ua, $auth, $extra) +}; + +(: split an absolute DB path into (collection, resource name) :) +declare %private function zotero:path-split($abs as xs:string) as map(*) { + let $name := tokenize($abs, "/")[last()] + let $coll := substring($abs, 1, string-length($abs) - string-length($name) - 1) + return map{"coll": $coll, "name": $name} +}; + +declare %private function zotero:resource-exists($coll as xs:string, $name as xs:string) as xs:boolean { + if (not(xmldb:collection-available($coll))) then false() + else some $r in xmldb:get-child-resources($coll) satisfies ($r = $name) +}; +(: read meta.json as JSON; create a template if missing — uses util:binary-doc :) +(:declare %private function zotero:read-meta() as map(*) { + let $ps := zotero:path-split($config:zotero-meta-path) + return + if (not(xmldb:collection-available($ps?coll))) then + map{"libraryVersion": 0, "syncedAt": ""} + else if (zotero:resource-exists($ps?coll, $ps?name)) then + let $bin := util:binary-doc($config:zotero-meta-path) + return try { parse-json(util:binary-to-string($bin)) } + catch * { map{"libraryVersion": 0, "syncedAt": ""} } + else ( + map{"libraryVersion": 0, "syncedAt": ""} + ) +};:) + +declare %private function zotero:read-meta() as map(*) { + let $ps := zotero:path-split($config:zotero-meta-path) + let $exists := zotero:resource-exists($ps?coll, $ps?name) + return + if (not($exists)) then + map{ "libraryVersion": 0, "syncedAt": "" } + else + let $uri := concat($ps?coll, "/", $ps?name) + let $bin := try { util:binary-doc($uri) } catch * { () } + let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" + return + if (normalize-space($txt) = "") then + map{ "libraryVersion": 0, "syncedAt": "" } + else + try { parse-json($txt) } catch * { map{ "libraryVersion": 0, "syncedAt": "" } } +}; + +(: write meta.json — 4-arg store to set media type :) +(:declare %private function zotero:write-meta($lmv as xs:integer) as xs:string { + let $ps := zotero:path-split($config:zotero-meta-path) + return + if (not(xmldb:collection-available($ps?coll))) then "" + else xmldb:store( + $ps?coll, $ps?name, + serialize(map{"libraryVersion": $lmv, "syncedAt": current-dateTime()}, map{"method":"json","indent": true()}), + "application/json" + ) +};:) +declare %private function zotero:write-meta($lmv as xs:integer) as xs:string { + let $ps := zotero:path-split($config:zotero-meta-path) + let $json := serialize( + map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, + map{ "method":"json", "indent": true() } + ) + return xmldb:store($ps?coll, $ps?name, $json, "application/json") +}; + +(: store one item .json into items dir — 4-arg store :) +declare %private function zotero:store-item($key as xs:string, $data as map(*)) as xs:string { + xmldb:store( + $config:zotero-items-dir, + concat($key, ".json"), + serialize($data, map{"method":"json","indent": true()}), + "application/json" + ) +}; + + +(: Ingest one page of items :) +declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { + let $n := + for $it in $arr?* + let $k := $it?key + let $d := $it?data + where exists($k) and exists($d) + return zotero:store-item($k, $d) + return count($arr?*) +}; + +(: returns exactly one xs:string: the rel="next" URL or '' :) +(:declare %private function zotero:next-link($resp as element(http:response)) as xs:string { + let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() + let $cands := + for $line in $links + let $parts := tokenize($line, ',') + for $p in $parts + where contains($p, 'rel="next"') or contains($p, "rel='next'") + let $u := normalize-space(substring-before(substring-after($p, '<'), '>')) + return $u + return string-join((($cands)[1]), '') :)(: coerce () → '' :)(: +};:) +declare %private function zotero:next-link($resp as element(http:response)) as xs:string { + let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() + let $cands := + for $line in $links + let $parts := tokenize($line, ',') + for $p in $parts + where contains($p, 'rel="next"') or contains($p, "rel='next'") + let $u := normalize-space(substring-before(substring-after($p, '<'), '>')) + return $u + return string-join((($cands)[1]), '') (: () -> '' :) +}; + +(: Follow pagination :) +(: follow pagination; $next may be empty :) +declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:integer) as xs:integer { + if (empty($next) or $next = '') then $acc + else + let $req := + + { zotero:headers(()) } + + let $seq := try { http:send-request($req) } catch * { () } + return + if (empty($seq)) then $acc + else + let $resp := $seq[1] + let $status := xs:integer($resp/@status) + return + if ($status != 200) then $acc + else + let $raw := try { util:binary-to-string($seq[2]) } catch * { "" } + let $arr := if (normalize-space($raw) = "") then array{} else + try { parse-json($raw) } catch * { array{} } + let $c := zotero:ingest-page($arr) + let $more := zotero:next-link($resp) + return zotero:sync-follow($more, $acc + $c) +}; + +(: ───────────────────────────────────────────────────────── + Incremental sync + ───────────────────────────────────────────────────────── :) + +(: primary worker: keep your existing sync($config,$root) unchanged :) +(: declare function zotero:sync($config as map(*), $root as element()) as xs:string { ... }; :) + +declare function zotero:debug-exports() as xs:string { + let $ns := "http://example.org/zotero" + let $ok2 := exists(function-lookup(QName($ns, "sync"), 2)) + let $ok1 := exists(function-lookup(QName($ns, "sync"), 1)) + let $ok0 := exists(function-lookup(QName($ns, "sync"), 0)) + return serialize(map{ + "sync#2": $ok2, "sync#1": $ok1, "sync#0": $ok0 + }, map{"method":"json","indent":true()}) +}; + + +(: ───────────────────────────────────────────────────────── + Public endpoint: POST /api/z/sync + ───────────────────────────────────────────────────────── :) +(: optional arity shims so Roaster can call #0/#1 too :) + +(: MAIN :) + +(: Roaster-safe wrappers; keep if your router may call #0/#1 :) +declare function zotero:sync() as xs:string { + response:set-header("Content-Type","application/json"), + zotero:sync(map{}, ) +}; +declare function zotero:sync($config as map(*)) as xs:string { + response:set-header("Content-Type","application/json"), + zotero:sync($config, ) +}; + +(: MAIN :) +declare function zotero:sync($config as map(*), $root as element()) as xs:string { + response:set-header("Content-Type","application/json"), + + let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } + let $since := xs:integer(($meta?libraryVersion, 0)[1]) + + let $base := concat($config:zotero-api-base, "/groups/", string($config:zotero-group-id), "/items") + (: IMPORTANT: '&' escaped as & in XML :) + let $href := concat($base, + "?since=", encode-for-uri(string($since)), + "&limit=100", + "&include=data") + + let $req := + + { + zotero:headers( + if ($since gt 0) + then + else () + ) + } + + + let $respSeq := try { http:send-request($req) } catch * { () } + + return serialize( + if (empty($respSeq)) then + map{ "status":"error", "reason":"http:send-request failed", "requestHref": $href } + else + let $resp := $respSeq[1] + let $status := xs:integer($resp/@status) + return + if ($status = 304) then + (: **WRITE META EVEN ON 304** to update syncedAt :) + let $_m := zotero:write-meta($since) + return map{ "status":"ok", "updated": 0, "libraryVersion": $since } + else if ($status != 200) then + let $errBody := try { util:binary-to-string($respSeq[2]) } catch * { "" } + return map{ + "status":"error", "httpStatus": $status, + "errorBody": $errBody, "requestHref": $href + } + else + let $raw := try { util:binary-to-string($respSeq[2]) } catch * { "" } + let $clean := if (starts-with($raw, codepoints-to-string(65279))) then substring($raw, 2) else $raw + let $arr := if (normalize-space($clean) = "") then array{} else + try { parse-json($clean) } catch * { array{} } + let $items := if ($arr instance of array(*)) then $arr else array{} + let $c1 := zotero:ingest-page($items) + + let $next := zotero:next-link($resp) + let $cN := if ($next = '') then 0 else zotero:sync-follow($next, 0) + + let $lmvStr := ($resp/http:header[lower-case(@name)='last-modified-version']/@value)[1] + let $lmv := if (exists($lmvStr) and normalize-space($lmvStr) ne "") then xs:integer($lmvStr) else $since + + (: **ALWAYS** persist the latest LMV & timestamp :) + let $_m := zotero:write-meta($lmv) + + return map{ "status":"ok", "updated": $c1 + $cN, "libraryVersion": $lmv } + , map{ "method":"json", "indent": true() }) +}; From d49cdc5a3c2efb528864bed921541a104f8803ce Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 9 Oct 2025 18:28:13 +0200 Subject: [PATCH 079/254] added zotero api with endpoint in custom-api. Syncs items to edep-data/zotero/groups/ID/items --- doc/zotero-cache.md | 9 +++- package-lock.json | 8 +--- src/modules/custom-api.json | 38 +++++++++++++++- src/modules/custom-api.xql | 1 + src/modules/lib/api.xql | 1 + src/modules/lib/api/zotero.xql | 83 ++++++++++++++++++++++++++++------ 6 files changed, 114 insertions(+), 26 deletions(-) diff --git a/doc/zotero-cache.md b/doc/zotero-cache.md index ad2b4b2..843d2d0 100644 --- a/doc/zotero-cache.md +++ b/doc/zotero-cache.md @@ -7,13 +7,18 @@ This document describes the local Zotero cache used by the app, the expected col --- +## Preconditions + +* you must install edep-data.xar BEFORE edep.xar so the latter can create the necessary structure (see below) + + ## 1) Collection layout Only the **group** collection is created dynamically by the post‑install script. The **base** and **items** collections are expected to exist after installation. ``` -/db -└── zotero-cache/ (collection) [pre-created] +/edep-data +└── zotero/ (collection) [pre-created] └── groups/ (collection) [pre-created] └── / (collection) [created by post-install] ├── items/ (collection) [created by post-install] diff --git a/package-lock.json b/package-lock.json index b513299..5403f9c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,7 @@ "dependencies": { "@jinntec/fore": "^2.6.0", "@jinntec/jinn-codemirror": "1.17.6", - "@teipublisher/pb-components": "2.12.10", - "datalist-ajax": "1.0.2" + "@teipublisher/pb-components": "2.12.10" }, "devDependencies": { "@eslint/js": "^9.36.0", @@ -4999,11 +4998,6 @@ "node": ">=0.10" } }, - "node_modules/datalist-ajax": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/datalist-ajax/-/datalist-ajax-1.0.2.tgz", - "integrity": "sha512-3rgGr9dGOIPked6INxjCUMENlHpZhokqpqKtssLrUY8p5jt82D+hdv+Ip4i+eahEYmIe0Hhf9E0hlRxTsUlSWw==" - }, "node_modules/dateformat": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index 46046d9..7682754 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -807,7 +807,7 @@ } } }, - "/api/document/{id}/epidoc": { + "/api/document/{id}/epidoc": { "get": { "summary": "Get the Epidoc source of a document", "description": "Get the source of a document, either as XML, text or binary.", @@ -849,7 +849,41 @@ } } } - } + }, + "/api/zotero/sync": { + "get": { + "summary": "Incrementally sync local cache.", + "tags": ["zotero"], + "operationId": "zotero:sync", + "responses": { + "200": { + "description": "Sync result.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "example": "ok" + }, + "updated": { + "type": "integer" + }, + "libraryVersion": { + "type": "integer" + } + } + } + } + } + }, + "304": { + "description": "No updates." + } + } + } + } }, "security": [ { diff --git a/src/modules/custom-api.xql b/src/modules/custom-api.xql index 694447e..2c7d041 100644 --- a/src/modules/custom-api.xql +++ b/src/modules/custom-api.xql @@ -13,6 +13,7 @@ import module namespace config="http://www.tei-c.org/tei-simple/config" at "conf import module namespace pm-config="http://www.tei-c.org/tei-simple/pm-config" at "pm-config.xql"; import module namespace tpu="http://www.tei-c.org/tei-publisher/util" at "lib/util.xql"; import module namespace errors = "http://e-editiones.org/roaster/errors"; +import module namespace zotero = "http://teipublisher.com/api/zotero" at "lib/api/zotero.xql"; declare namespace json="http://www.json.org"; declare namespace tei="http://www.tei-c.org/ns/1.0"; diff --git a/src/modules/lib/api.xql b/src/modules/lib/api.xql index ce2eb3f..066976e 100644 --- a/src/modules/lib/api.xql +++ b/src/modules/lib/api.xql @@ -17,6 +17,7 @@ import module namespace vapi="http://teipublisher.com/api/view" at "api/view.xql import module namespace anno="http://teipublisher.com/api/annotations" at "api/annotations.xql"; import module namespace custom="http://teipublisher.com/api/custom" at "../custom-api.xql"; import module namespace nlp="http://teipublisher.com/api/nlp" at "api/nlp.xql"; +import module namespace zotero="http://teipublisher.com/api/zotero" at "api/zotero.xql"; declare option output:indent "no"; diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 156587a..36d1258 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -53,10 +53,17 @@ declare %private function zotero:headers($extra as element(http:header)*) as ele }; (: split an absolute DB path into (collection, resource name) :) -declare %private function zotero:path-split($abs as xs:string) as map(*) { +(:declare %private function zotero:path-split($abs as xs:string) as map(*) { let $name := tokenize($abs, "/")[last()] let $coll := substring($abs, 1, string-length($abs) - string-length($name) - 1) return map{"coll": $coll, "name": $name} +};:) +(: ───────────────── path helper (normalized) ───────────────── :) +declare %private function zotero:path-split($abs as xs:string) as map(*) { + let $norm := replace($abs, '/+$', '') (: drop trailing '/' :) + let $name := tokenize($norm, '/')[last()] + let $coll := substring($norm, 1, string-length($norm) - string-length($name) - 1) + return map{ "coll": $coll, "name": $name } }; declare %private function zotero:resource-exists($coll as xs:string, $name as xs:string) as xs:boolean { @@ -106,14 +113,35 @@ declare %private function zotero:read-meta() as map(*) { "application/json" ) };:) +(: overwrite meta.json with new libraryVersion + syncedAt :) +(: declare %private function zotero:write-meta($lmv as xs:integer) as xs:string { let $ps := zotero:path-split($config:zotero-meta-path) + let $_rm := try { xmldb:remove($ps?coll, $ps?name) } catch * { () } let $json := serialize( - map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, - map{ "method":"json", "indent": true() } - ) + map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, + map{ "method":"json", "indent": true() } + ) return xmldb:store($ps?coll, $ps?name, $json, "application/json") }; +:) + +(: ─────────── overwrite meta.json; return true() on success ─────────── :) +declare %private function zotero:write-meta($lmv as xs:integer) as xs:boolean { + let $ps := zotero:path-split($config:zotero-meta-path) + let $json := serialize( + map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, + map{ "method":"json", "indent": true() } + ) + let $_rm := try { xmldb:remove($ps?coll, $ps?name) } catch * { () } + return + try { + let $_ := xmldb:store($ps?coll, $ps?name, $json, "application/json") + return true() + } catch * { + false() + } +}; (: store one item .json into items dir — 4-arg store :) declare %private function zotero:store-item($key as xs:string, $data as map(*)) as xs:string { @@ -223,6 +251,8 @@ declare function zotero:sync($config as map(*)) as xs:string { }; (: MAIN :) +(: INLINE sync — writes meta.json on BOTH 304 and 200 :) +(: ─────────── sync: ALWAYS writes meta.json (200 and 304) ─────────── :) declare function zotero:sync($config as map(*), $root as element()) as xs:string { response:set-header("Content-Type","application/json"), @@ -230,7 +260,6 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string let $since := xs:integer(($meta?libraryVersion, 0)[1]) let $base := concat($config:zotero-api-base, "/groups/", string($config:zotero-group-id), "/items") - (: IMPORTANT: '&' escaped as & in XML :) let $href := concat($base, "?since=", encode-for-uri(string($since)), "&limit=100", @@ -251,20 +280,33 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string return serialize( if (empty($respSeq)) then - map{ "status":"error", "reason":"http:send-request failed", "requestHref": $href } + map{ + "status" : "error", + "reason" : "http:send-request failed", + "requestHref" : $href, + "metaPath" : $config:zotero-meta-path + } else let $resp := $respSeq[1] let $status := xs:integer($resp/@status) return if ($status = 304) then - (: **WRITE META EVEN ON 304** to update syncedAt :) - let $_m := zotero:write-meta($since) - return map{ "status":"ok", "updated": 0, "libraryVersion": $since } + let $ok := zotero:write-meta($since) + return map{ + "status" : "ok", + "updated" : 0, + "libraryVersion" : $since, + "metaWriteOk" : $ok, + "metaPath" : $config:zotero-meta-path + } else if ($status != 200) then let $errBody := try { util:binary-to-string($respSeq[2]) } catch * { "" } return map{ - "status":"error", "httpStatus": $status, - "errorBody": $errBody, "requestHref": $href + "status" : "error", + "httpStatus" : $status, + "errorBody" : $errBody, + "requestHref" : $href, + "metaPath" : $config:zotero-meta-path } else let $raw := try { util:binary-to-string($respSeq[2]) } catch * { "" } @@ -274,15 +316,26 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string let $items := if ($arr instance of array(*)) then $arr else array{} let $c1 := zotero:ingest-page($items) - let $next := zotero:next-link($resp) + let $next := ( + for $line in $resp/http:header[lower-case(@name)='link']/@value/string() + let $parts := tokenize($line, ",") + for $p in $parts + where contains($p, 'rel="next"') or contains($p, "rel='next'") + return normalize-space(substring-before(substring-after($p, "<"), ">")) + )[1] let $cN := if ($next = '') then 0 else zotero:sync-follow($next, 0) let $lmvStr := ($resp/http:header[lower-case(@name)='last-modified-version']/@value)[1] let $lmv := if (exists($lmvStr) and normalize-space($lmvStr) ne "") then xs:integer($lmvStr) else $since - (: **ALWAYS** persist the latest LMV & timestamp :) - let $_m := zotero:write-meta($lmv) + let $ok := zotero:write-meta($lmv) - return map{ "status":"ok", "updated": $c1 + $cN, "libraryVersion": $lmv } + return map{ + "status" : "ok", + "updated" : $c1 + $cN, + "libraryVersion" : $lmv, + "metaWriteOk" : $ok, + "metaPath" : $config:zotero-meta-path + } , map{ "method":"json", "indent": true() }) }; From 1bfe92b1806e9d0911e3ee74bf505b751e364e67 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 10 Oct 2025 09:57:13 +0200 Subject: [PATCH 080/254] cleanup --- src/modules/lib/api/zotero.xql | 77 +--------------------------------- 1 file changed, 1 insertion(+), 76 deletions(-) diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 36d1258..0f4ef0d 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -7,9 +7,7 @@ declare namespace http = "http://expath.org/ns/http-client"; declare namespace xmldb = "http://exist-db.org/xquery/xmldb"; declare namespace util = "http://exist-db.org/xquery/util"; import module namespace response = "http://exist-db.org/xquery/response"; -(: Import your config module — update the path :) -import module namespace config = "http://www.tei-c.org/tei-simple/config" - at "../../config.xqm"; +import module namespace config = "http://www.tei-c.org/tei-simple/config" at "../../config.xqm"; (: ───────────────────────────────────────────────────────── Helpers @@ -19,21 +17,6 @@ declare %private function zotero:json($v as item()*) as xs:string { serialize($v, map{"method":"json","indent":true()}) }; -(: Build headers; Authorization omitted if key is empty :) -(: -declare %private function zotero:headers($extra as element(http:header)*) as element(http:header)* { - let $apiVer := - let $accept := - let $ua := - let $auth := - if (normalize-space($config:zotero-api-key) ne "") then ( - , - - ) else () - return ($apiVer, $accept, $ua, $auth, $extra) -}; -:) - (: grab header values case-insensitively :) declare %private function zotero:header($resp as element(http:response), $name as xs:string) as xs:string* { $resp/http:header[lower-case(@name) = lower-case($name)]/@value/string() @@ -52,12 +35,6 @@ declare %private function zotero:headers($extra as element(http:header)*) as ele return ($apiVer, $accept, $ua, $auth, $extra) }; -(: split an absolute DB path into (collection, resource name) :) -(:declare %private function zotero:path-split($abs as xs:string) as map(*) { - let $name := tokenize($abs, "/")[last()] - let $coll := substring($abs, 1, string-length($abs) - string-length($name) - 1) - return map{"coll": $coll, "name": $name} -};:) (: ───────────────── path helper (normalized) ───────────────── :) declare %private function zotero:path-split($abs as xs:string) as map(*) { let $norm := replace($abs, '/+$', '') (: drop trailing '/' :) @@ -70,20 +47,6 @@ declare %private function zotero:resource-exists($coll as xs:string, $name as xs if (not(xmldb:collection-available($coll))) then false() else some $r in xmldb:get-child-resources($coll) satisfies ($r = $name) }; -(: read meta.json as JSON; create a template if missing — uses util:binary-doc :) -(:declare %private function zotero:read-meta() as map(*) { - let $ps := zotero:path-split($config:zotero-meta-path) - return - if (not(xmldb:collection-available($ps?coll))) then - map{"libraryVersion": 0, "syncedAt": ""} - else if (zotero:resource-exists($ps?coll, $ps?name)) then - let $bin := util:binary-doc($config:zotero-meta-path) - return try { parse-json(util:binary-to-string($bin)) } - catch * { map{"libraryVersion": 0, "syncedAt": ""} } - else ( - map{"libraryVersion": 0, "syncedAt": ""} - ) -};:) declare %private function zotero:read-meta() as map(*) { let $ps := zotero:path-split($config:zotero-meta-path) @@ -102,30 +65,6 @@ declare %private function zotero:read-meta() as map(*) { try { parse-json($txt) } catch * { map{ "libraryVersion": 0, "syncedAt": "" } } }; -(: write meta.json — 4-arg store to set media type :) -(:declare %private function zotero:write-meta($lmv as xs:integer) as xs:string { - let $ps := zotero:path-split($config:zotero-meta-path) - return - if (not(xmldb:collection-available($ps?coll))) then "" - else xmldb:store( - $ps?coll, $ps?name, - serialize(map{"libraryVersion": $lmv, "syncedAt": current-dateTime()}, map{"method":"json","indent": true()}), - "application/json" - ) -};:) -(: overwrite meta.json with new libraryVersion + syncedAt :) -(: -declare %private function zotero:write-meta($lmv as xs:integer) as xs:string { - let $ps := zotero:path-split($config:zotero-meta-path) - let $_rm := try { xmldb:remove($ps?coll, $ps?name) } catch * { () } - let $json := serialize( - map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, - map{ "method":"json", "indent": true() } - ) - return xmldb:store($ps?coll, $ps?name, $json, "application/json") -}; -:) - (: ─────────── overwrite meta.json; return true() on success ─────────── :) declare %private function zotero:write-meta($lmv as xs:integer) as xs:boolean { let $ps := zotero:path-split($config:zotero-meta-path) @@ -165,18 +104,6 @@ declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { return count($arr?*) }; -(: returns exactly one xs:string: the rel="next" URL or '' :) -(:declare %private function zotero:next-link($resp as element(http:response)) as xs:string { - let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() - let $cands := - for $line in $links - let $parts := tokenize($line, ',') - for $p in $parts - where contains($p, 'rel="next"') or contains($p, "rel='next'") - let $u := normalize-space(substring-before(substring-after($p, '<'), '>')) - return $u - return string-join((($cands)[1]), '') :)(: coerce () → '' :)(: -};:) declare %private function zotero:next-link($resp as element(http:response)) as xs:string { let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() let $cands := @@ -189,7 +116,6 @@ declare %private function zotero:next-link($resp as element(http:response)) as x return string-join((($cands)[1]), '') (: () -> '' :) }; -(: Follow pagination :) (: follow pagination; $next may be empty :) declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:integer) as xs:integer { if (empty($next) or $next = '') then $acc @@ -220,7 +146,6 @@ declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:int ───────────────────────────────────────────────────────── :) (: primary worker: keep your existing sync($config,$root) unchanged :) -(: declare function zotero:sync($config as map(*), $root as element()) as xs:string { ... }; :) declare function zotero:debug-exports() as xs:string { let $ns := "http://example.org/zotero" From b375fa88dba5829304c8ffde9be2d478780f2aa9 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 10 Oct 2025 10:12:04 +0200 Subject: [PATCH 081/254] needs an re-index --- src/post-install.xql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/post-install.xql b/src/post-install.xql index f7a571d..57713e0 100644 --- a/src/post-install.xql +++ b/src/post-install.xql @@ -201,6 +201,7 @@ local:mkcol($target, "transform"), local:generate-code($target), local:create-data-collection(), local:zotero-ensure-layout(), +xmldb:reindex('/db/apps/edep-data'), let $pmuConfig := pmc:generate-pm-config(($config:odd-available, $config:odd-internal), $config:default-odd, $config:odd-root) return xmldb:store($config:app-root || "/modules", "pm-config.xql", $pmuConfig, "application/xquery") \ No newline at end of file From b59f65880898313d5f29e5751506c8c0bd9800cc Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 10 Oct 2025 11:20:09 +0200 Subject: [PATCH 082/254] added config param for 'style' param, including bib in cached data --- src/modules/config.xqm | 1 + src/modules/lib/api/zotero.xql | 35 ++++++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/modules/config.xqm b/src/modules/config.xqm index 33a0c1b..6a1efa8 100644 --- a/src/modules/config.xqm +++ b/src/modules/config.xqm @@ -346,6 +346,7 @@ declare variable $config:zotero-base-dir := $config:data-root || "/zotero/groups declare variable $config:zotero-group-dir := $config:zotero-base-dir || "/" || $config:zotero-group-id; declare variable $config:zotero-items-dir := $config:zotero-group-dir || "/items"; declare variable $config:zotero-meta-path := $config:zotero-group-dir || "/meta.json"; +declare variable $config:zotero-style := "digital-humanities-im-deutschsprachigen-raum"; (:~ diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 0f4ef0d..a0fb1d6 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -93,17 +93,30 @@ declare %private function zotero:store-item($key as xs:string, $data as map(*)) }; -(: Ingest one page of items :) +(: Ingest one page of items including bib :) declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { - let $n := - for $it in $arr?* - let $k := $it?key - let $d := $it?data - where exists($k) and exists($d) - return zotero:store-item($k, $d) - return count($arr?*) + let $n := array:size($arr) + return + if ($n = 0) then 0 + else + sum( + for $i in 1 to $n + let $item := array:get($arr, $i) + let $key := string(($item?key, $item?data?key)[1]) + let $data := $item?data + let $bib := $item?bib + let $toSave := + if (exists($data)) then + if (exists($bib)) then map:merge(($data, map{"bib": string($bib)})) + else $data + else + (: extremely rare, but if Zotero returned only a bib :) + map{"bib": string($bib)} + let $_ := + if ($key != "") then zotero:store-item($key, $toSave) else () + return if ($key != "") then 1 else 0 + ) }; - declare %private function zotero:next-link($resp as element(http:response)) as xs:string { let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() let $cands := @@ -188,7 +201,9 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string let $href := concat($base, "?since=", encode-for-uri(string($since)), "&limit=100", - "&include=data") + "&include=data,bib", + "&format=json", + "&style=$config:zotero-style") let $req := From e5397dbb675033d802e86690d68b98ae08e69007 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 10 Oct 2025 12:31:18 +0200 Subject: [PATCH 083/254] return proper json (bit hacky to circumwent roaster here) --- src/modules/custom-api.json | 19 +------------------ src/modules/lib/api/zotero.xql | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index 7682754..119803b 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -859,24 +859,7 @@ "200": { "description": "Sync result.", "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "status": { - "type": "string", - "example": "ok" - }, - "updated": { - "type": "integer" - }, - "libraryVersion": { - "type": "integer" - } - } - } - } - } + "text/plain": { "schema": { "type": "string" } } } }, "304": { "description": "No updates." diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index a0fb1d6..b768f92 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -191,19 +191,23 @@ declare function zotero:sync($config as map(*)) as xs:string { (: MAIN :) (: INLINE sync — writes meta.json on BOTH 304 and 200 :) (: ─────────── sync: ALWAYS writes meta.json (200 and 304) ─────────── :) -declare function zotero:sync($config as map(*), $root as element()) as xs:string { +declare function zotero:sync($config as map(*), $root as element()) { response:set-header("Content-Type","application/json"), let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } let $since := xs:integer(($meta?libraryVersion, 0)[1]) let $base := concat($config:zotero-api-base, "/groups/", string($config:zotero-group-id), "/items") - let $href := concat($base, - "?since=", encode-for-uri(string($since)), - "&limit=100", - "&include=data,bib", - "&format=json", - "&style=$config:zotero-style") + + (: IMPORTANT: & must be & inside attributes; style value must be encoded :) + let $href := concat( + $base, + "?since=", encode-for-uri(string($since)), + "&limit=100", + "&include=data,bib", + "&format=json", + "&style=",$config:zotero-style + ) let $req := @@ -218,7 +222,7 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string let $respSeq := try { http:send-request($req) } catch * { () } - return serialize( + let $payload := if (empty($respSeq)) then map{ "status" : "error", @@ -256,13 +260,15 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string let $items := if ($arr instance of array(*)) then $arr else array{} let $c1 := zotero:ingest-page($items) - let $next := ( + (: make $next a STRING so we never pass () into sync-follow :) + let $next := string(( for $line in $resp/http:header[lower-case(@name)='link']/@value/string() let $parts := tokenize($line, ",") for $p in $parts where contains($p, 'rel="next"') or contains($p, "rel='next'") return normalize-space(substring-before(substring-after($p, "<"), ">")) - )[1] + )[1]) + let $cN := if ($next = '') then 0 else zotero:sync-follow($next, 0) let $lmvStr := ($resp/http:header[lower-case(@name)='last-modified-version']/@value)[1] @@ -277,5 +283,7 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string "metaWriteOk" : $ok, "metaPath" : $config:zotero-meta-path } - , map{ "method":"json", "indent": true() }) + + return serialize($payload, map{ "method":"json", "indent": true() }) }; + From c3e74528e95ce00fa856eeef29aa8891602f074a Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 10 Oct 2025 15:15:31 +0200 Subject: [PATCH 084/254] implemenation of new zotero search endpoint against locally stored items --- src/modules/custom-api.json | 155 +++++++++++++++++++++------------ src/modules/lib/api/zotero.xql | 83 ++++++++++++++++++ 2 files changed, 184 insertions(+), 54 deletions(-) diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index 119803b..c3e5f11 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -808,65 +808,112 @@ } }, "/api/document/{id}/epidoc": { - "get": { - "summary": "Get the Epidoc source of a document", - "description": "Get the source of a document, either as XML, text or binary.", - "tags": ["documents"], - "operationId": "dapi:epidoc", - "parameters": [ - { - "name": "id", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Content of the document", - "content": { - "application/xml": { - "schema": { - "type": "string" - } - }, - "text/markdown": { - "schema": { - "type": "string" - } - }, - "text/text": { - "schema": { - "type": "string" - } - } - } - }, - "410": { - "description": "Document deleted" - } - } - } - }, - "/api/zotero/sync": { - "get": { - "summary": "Incrementally sync local cache.", - "tags": ["zotero"], - "operationId": "zotero:sync", + "get": { + "summary": "Get the Epidoc source of a document", + "description": "Get the source of a document, either as XML, text or binary.", + "tags": ["documents"], + "operationId": "dapi:epidoc", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { - "200": { - "description": "Sync result.", - "content": { - "text/plain": { "schema": { "type": "string" } } } + "200": { + "description": "Content of the document", + "content": { + "application/xml": { + "schema": { + "type": "string" + } + }, + "text/markdown": { + "schema": { + "type": "string" + } + }, + "text/text": { + "schema": { + "type": "string" + } + } + } + }, + "410": { + "description": "Document deleted" + } + } + } + }, + "/api/zotero/sync": { + "get": { + "summary": "Incrementally sync local cache.", + "tags": ["zotero"], + "operationId": "zotero:sync", + "responses": { + "200": { + "description": "Sync result.", + "content": { + "text/plain": { "schema": { "type": "string" } } + } + }, + "304": { + "description": "No updates." + } + } + } + }, + "/api/zotero/items/search": { + "get": { + "summary": "Search cached items. JSON only.", + "tags": ["zotero"], + "operationId": "zotero:items-search", + "parameters": [ + { + "in": "query", + "name": "q", + "schema": { + "type": "string", + "default": "Ama" + }, + "description": "Full-text over title/creators/DOI." + }, + { + "in": "query", + "name": "tag", + "schema": { + "type": "string" + }, + "description": "Restrict to items having this tag." }, - "304": { - "description": "No updates." + { + "in": "query", + "name": "limit", + "schema": { + "type": "integer", + "default": 15 + } + } + ], + "responses": { + "200": { + "description": "Array of items (key + data).", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } } - } } } + } }, "security": [ { diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index b768f92..6dc3798 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -287,3 +287,86 @@ declare function zotero:sync($config as map(*), $root as element()) { return serialize($payload, map{ "method":"json", "indent": true() }) }; +(: ─── Wrappers so Roaster can resolve any arity ─── :) +declare function zotero:items-search() as xs:string { + zotero:items-search(map{}, ) +}; + +declare function zotero:items-search($config as map(*)) as xs:string { + zotero:items-search($config, ) +}; + +(: ─── MAIN: GET /api/zotero/items/search ─── :) +declare function zotero:items-search($config as map(*), $root as element()) as xs:string { + response:set-header("Content-Type", "application/json"), + + let $coll := $config:zotero-items-dir + let $qIn := lower-case(normalize-space(request:get-parameter("q", ""))) + let $tagIn := lower-case(normalize-space(request:get-parameter("tag", ""))) + let $limIn := request:get-parameter("limit", "15") + let $limit := let $n := try { xs:integer($limIn) } catch * { 15 } + return if ($n lt 1) then 15 else $n + + let $names := + if (xmldb:collection-available($coll)) + then for $n in xmldb:get-child-resources($coll) + where ends-with($n, ".json") + return $n + else () + + let $matches := + for $name in $names + let $uri := concat($coll, "/", $name) + let $bin := try { util:binary-doc($uri) } catch * { () } + let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" + where string-length($txt) gt 0 + let $data := try { parse-json($txt) } catch * { map{} } + + let $key := string(( $data?key, replace($name, "\.json$", "") )[1]) + + (: tag filter :) + let $hasTag := + if ($tagIn = "") then true() + else if ($data?tags instance of array(*)) then + some $i in 1 to array:size($data?tags) + satisfies lower-case(string((array:get($data?tags, $i)?tag)[1])) = $tagIn + else false() + + (: q filter over title + creators + DOI — SAFE coalescing :) + let $title := lower-case(string(($data?title)[1])) + let $creators := + if ($data?creators instance of array(*)) then + string-join( + for $i in 1 to array:size($data?creators) + let $c := array:get($data?creators, $i) + let $ln := string(($c?lastName)[1]) + let $fn := string(($c?firstName)[1]) + let $nm := string(($c?name)[1]) + let $parts := ($ln, $fn, $nm) + let $nonEmpty := for $p in $parts where normalize-space($p) ne "" return $p + let $one := normalize-space(string-join($nonEmpty, " ")) + where $one ne "" + return $one + , " ") + else "" + let $doi := lower-case(string((($data?DOI, $data?doi)[1]))) + + let $hay := normalize-space(string-join(($title, $creators, $doi), " ")) + let $okQ := ($qIn = "") or contains($hay, $qIn) + + where $hasTag and $okQ + return map{ "key": $key, "data": $data } + + let $total := count($matches) + let $limited := subsequence($matches, 1, $limit) + let $_hdr := response:set-header("X-Total-Count", string($total)) + + let $payload := map{ + "query": map{ "q": $qIn, "tag": $tagIn, "limit": $limit }, + "total": $total, (: matches before limit :) + "returned": count($limited), (: items in this page :) + "items": array { $limited } (: [{key,data}, …] :) + } + + return serialize($payload, map{ "method": "json", "indent": true() }) +}; From 8bffc4369ccb6de86a9a72cbbee73fa0ee83b04e Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Sat, 11 Oct 2025 12:51:47 +0200 Subject: [PATCH 085/254] refactored parameter name --- src/modules/lib/api/zotero.xql | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 6dc3798..9ddef86 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -183,15 +183,15 @@ declare function zotero:sync() as xs:string { response:set-header("Content-Type","application/json"), zotero:sync(map{}, ) }; -declare function zotero:sync($config as map(*)) as xs:string { +declare function zotero:sync($request as map(*)) as xs:string { response:set-header("Content-Type","application/json"), - zotero:sync($config, ) + zotero:sync($request, ) }; (: MAIN :) (: INLINE sync — writes meta.json on BOTH 304 and 200 :) (: ─────────── sync: ALWAYS writes meta.json (200 and 304) ─────────── :) -declare function zotero:sync($config as map(*), $root as element()) { +declare function zotero:sync($request as map(*), $root as element()) { response:set-header("Content-Type","application/json"), let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } @@ -292,18 +292,18 @@ declare function zotero:items-search() as xs:string { zotero:items-search(map{}, ) }; -declare function zotero:items-search($config as map(*)) as xs:string { - zotero:items-search($config, ) +declare function zotero:items-search($request as map(*)) as xs:string { + zotero:items-search($request, ) }; (: ─── MAIN: GET /api/zotero/items/search ─── :) -declare function zotero:items-search($config as map(*), $root as element()) as xs:string { +declare function zotero:items-search($request as map(*), $root as element()) as xs:string { response:set-header("Content-Type", "application/json"), let $coll := $config:zotero-items-dir let $qIn := lower-case(normalize-space(request:get-parameter("q", ""))) let $tagIn := lower-case(normalize-space(request:get-parameter("tag", ""))) - let $limIn := request:get-parameter("limit", "15") + let $limIn := $request?parameters?limit let $limit := let $n := try { xs:integer($limIn) } catch * { 15 } return if ($n lt 1) then 15 else $n @@ -370,3 +370,4 @@ declare function zotero:items-search($config as map(*), $root as element()) as x return serialize($payload, map{ "method": "json", "indent": true() }) }; + From 2898ed6d329de05e7646b41f8b1a5ed6170f1418 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Sat, 11 Oct 2025 17:25:38 +0200 Subject: [PATCH 086/254] implemented api/zotero/items/bib endpoint, simple test page zotero.html, documentation --- doc/zotero-cache.md | 29 +- src/expath-pkg.xml | 1 + src/modules/custom-api.json | 19 ++ src/modules/lib/api/zotero.xql | 472 +++++++++++++++++++++++++++++---- src/repo.xml | 2 +- src/templates/zotero.html | 200 ++++++++++++++ 6 files changed, 665 insertions(+), 58 deletions(-) create mode 100644 src/templates/zotero.html diff --git a/doc/zotero-cache.md b/doc/zotero-cache.md index 843d2d0..47b70be 100644 --- a/doc/zotero-cache.md +++ b/doc/zotero-cache.md @@ -11,10 +11,18 @@ This document describes the local Zotero cache used by the app, the expected col * you must install edep-data.xar BEFORE edep.xar so the latter can create the necessary structure (see below) +## What is does -## 1) Collection layout +* Syncs the items of a specific configured group to a eXist-db collection including its `bib` properties. +* maintains a meta.json to track version -Only the **group** collection is created dynamically by the post‑install script. The **base** and **items** collections are expected to exist after installation. + +## Collection layout + +The following structure will be created during post-install. + +Note: Currently only one group is supported for syncing. However the structure allows to store +multiple groups with their items. ``` /edep-data @@ -54,7 +62,7 @@ One file per Zotero item **key**. Stored content is the **pristine Zotero `data` --- -## 2) Config module (`config.xqm`) +## Config module (`config.xqm`) The application reads **config variables** directly (hyphenated names), and also provides **function wrappers** for legacy callers. Keep both to avoid router/package regressions. @@ -71,6 +79,7 @@ declare variable $config:zotero-base-dir as xs:string := "/db/zotero-cache/gro declare variable $config:zotero-group-dir as xs:string := concat($config:zotero-base-dir, "/", $config:zotero-group-id); declare variable $config:zotero-items-dir as xs:string := concat($config:zotero-group-dir, "/items"); declare variable $config:zotero-meta-path as xs:string := concat($config:zotero-group-dir, "/meta.json"); +declare variable $config:zotero-style := "digital-humanities-im-deutschsprachigen-raum"; ``` **Notes** @@ -79,7 +88,7 @@ declare variable $config:zotero-meta-path as xs:string := concat($config:zotero --- -## 3) Installation bootstrap (`post-install.xql`) +## Installation bootstrap (`post-install.xql`) The post‑install script should: 1. Create collections: `$config:zotero-base-dir`, `$config:zotero-group-dir`, `$config:zotero-items-dir` (stepwise under `/db`). @@ -94,7 +103,7 @@ A minimal post‑install does: --- -## 4) Sync endpoint +## Sync endpoint **Route**: `POST /api/z/sync` → `zotero:sync` **Behavior**: Incremental, paginated sync from Zotero group into local cache. @@ -129,7 +138,7 @@ If Zotero returns `304 Not Modified`, the local API responds with: --- -## 5) Router wiring (Roaster) +## Router wiring (Roaster) Example OpenAPI snippet (YAML): ```yaml @@ -160,7 +169,7 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string --- -## 6) Troubleshooting +## Troubleshooting - **make sure edep-data.xar has been installed before edep.xar** post-install of edep.xar creates the needed collections in edep-data @@ -176,7 +185,7 @@ declare function zotero:sync($config as map(*), $root as element()) as xs:string --- -## 7) Trigger sync (curl) +## Trigger sync (curl) ```bash curl -X POST 'http://localhost:8080/api/z/sync' ``` @@ -186,7 +195,3 @@ You should see JSON like: { "status":"ok", "updated": 123, "libraryVersion": 4567 } ``` ---- - -*Document version:* 1.0 -*Last updated:* generated for the current build. diff --git a/src/expath-pkg.xml b/src/expath-pkg.xml index 75d5c71..0b9cad8 100644 --- a/src/expath-pkg.xml +++ b/src/expath-pkg.xml @@ -3,6 +3,7 @@ Editionstools für eine digitale Epigraphik + diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index c3e5f11..2d901d6 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -868,6 +868,24 @@ } } }, + "/api/zotero/items/bib": { + "get": { + "summary": "Return a single cached item's bibliography (HTML) by key or tag.", + "tags": ["zotero"], + "operationId": "zotero:item-bib", + "parameters": [ + { "in": "query", "name": "key", "schema": { "type": "string" }, "description": "Item key (preferred)." }, + { "in": "query", "name": "tag", "schema": { "type": "string" }, "description": "Fallback: first item with this tag." } + ], + "responses": { + "200": { + "description": "HTML snippet (CSL bib or title fallback).", + "content": { "text/html": { "schema": { "type": "string" } } } + }, + "404": { "description": "No matching item found." } + } + } + }, "/api/zotero/items/search": { "get": { "summary": "Search cached items. JSON only.", @@ -914,6 +932,7 @@ } } } + }, "security": [ { diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 9ddef86..b9c8f90 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -9,9 +9,106 @@ declare namespace util = "http://exist-db.org/xquery/util"; import module namespace response = "http://exist-db.org/xquery/response"; import module namespace config = "http://www.tei-c.org/tei-simple/config" at "../../config.xqm"; -(: ───────────────────────────────────────────────────────── - Helpers - ───────────────────────────────────────────────────────── :) +(:~ + ============================================================================== + Zotero cache + lookup module + ============================================================================== + + Purpose + ------- + Provide a thin caching layer for a single Zotero group and a set of read APIs + that the Fore UI can call with low latency. The module: + • syncs items from the Zotero Web API (data + bib), + • stores each record as .json in the local DB, + • exposes lightweight endpoints for search and for rendering a single + bibliography (or a safe title fallback) as HTML. + + Public endpoints (Roaster) + -------------------------- + Each endpoint exposes 3 arities so Roaster can bind it: + f(), f($request as map(*)), f($request as map(*), $root as element()). + + 1) zotero:sync($request, $root) as xs:string + - Fetches from Zotero /groups/{groupId}/items with include=data,bib and + the configured CSL style; follows Link rel="next". + - Writes items to $config:zotero-items-dir as .json. + - Updates meta.json with libraryVersion and syncedAt. + - Returns a serialized JSON string (xs:string). Sets Content-Type: application/json. + - NOTE: We serialize on purpose to avoid Roaster atomizing map/array + values (FOTY0013). If Roaster is patched later, this can return map(*). + + 2) zotero:items-search($request, $root) as xs:string + - Query params: q (full-text over title + creators + DOI), tag (exact, + case-insensitive), limit (default 15). + - Returns serialized JSON with { query, total, returned, items[] }. + - Sets headers: Content-Type: application/json, X-Total-Count: . + + 3) zotero:item-bib-top($request, $root) as xs:string + - Query params: + key = item key; OR + tag = tag name (top-level only, first match). + - Returns a single HTML snippet: + • bib string if present, else + • Title or note (safely serialized). + - Sets Content-Type: text/html; charset=UTF-8. + + Configuration (from modules/config.xqm) + --------------------------------------- + The module expects these variables to be provided by your app config: + + $config:zotero-api-base xs:string + Base URL for Zotero Web API, e.g. "https://api.zotero.org". + + $config:zotero-group-id xs:string or xs:integer + The single group to sync, e.g. "2519759". + + $config:zotero-style xs:string + CSL style id for server-side bibliography rendering, e.g. + "chicago-note-bibliography" or "digital-humanities-im-deutschsprachigen-raum". + + $config:zotero-items-dir xs:string + Collection where item JSON files are stored, e.g. + "/db/apps/edep-data/zotero/groups/2519759/items". + + $config:zotero-meta-path xs:string + Full resource path to meta.json, e.g. + "/db/apps/edep-data/zotero/groups/2519759/meta.json". + + $config:zotero-api-key xs:string (optional) + API key if the group requires auth. Public groups can omit this. + + Storage layout (created by post-install) + ---------------------------------------- + /db/apps/edep-data/zotero/ + groups/ + {groupId}/ + items/ + .json (serialized map { data: {...}, bib: "" }) + meta.json (serialized map { libraryVersion: int, syncedAt: dateTime }) + + Error handling and status + ------------------------- + • Upstream HTTP errors return a JSON body { status: "error", httpStatus, ... }. + • Sync handles 304 Not Modified and updates meta.json accordingly. + • Search always returns 200 with an empty result set when nothing matches. + • item-bib-top returns: + 200 on success, + 400 when both key and tag are missing, + 404 when the cache is missing or no item was found. + + Notes on Roaster JSON handling + ------------------------------ + Roaster 1.10.0 atomizes function results; XDM maps/arrays are function + items and cannot be atomized (FOTY0013). Until Roaster gains native JSON + output for map/array values, endpoints in this module serialize their + responses to xs:string and set the Content-Type header themselves. + In OpenAPI, you can declare "text/plain" for such responses to avoid + double-encoding; clients still see valid JSON because the handler sets + application/json. + + ============================================================================== +:) + declare %private function zotero:json($v as item()*) as xs:string { serialize($v, map{"method":"json","indent":true()}) @@ -22,19 +119,51 @@ declare %private function zotero:header($resp as element(http:response), $name a $resp/http:header[lower-case(@name) = lower-case($name)]/@value/string() }; -(: replace your zotero:headers :) -declare %private function zotero:headers($extra as element(http:header)*) as element(http:header)* { - let $apiVer := - let $accept := - let $ua := - let $auth := - if (normalize-space($config:zotero-api-key) ne "") then ( - , - - ) else () - return ($apiVer, $accept, $ua, $auth, $extra) +(:~ + Build standard HTTP headers for Zotero API calls. + + Always includes: + - Accept: application/json + - User-Agent: eXist/zotero-sync + + Optionally includes: + - Authorization / Zotero-API-Key header if configured in your config.xqm. + - Any extra header passed in (e.g., If-Modified-Since-Version). + + Parameters: + @param $extra element(http:header)? Optional extra header to include. + + Return: + @return element(http:header)* A sequence of http:header elements ready to + be inserted into . + + Example: + + { attribute href { $href } } + { zotero:headers( + if ($since gt 0) + then + else () + ) + } + +:) +(: Build headers for Zotero requests :) +declare function zotero:headers($extra as element(http:header)?) as element(http:header)* { + let $base := + (, + , + (: avoid gzip auto-encoding issues :) + ) + let $key := normalize-space(string(($config:zotero-api-key, "")[1])) + let $auth := + if ($key ne "") + then + else () + return ($base, $auth, $extra) }; + (: ───────────────── path helper (normalized) ───────────────── :) declare %private function zotero:path-split($abs as xs:string) as map(*) { let $norm := replace($abs, '/+$', '') (: drop trailing '/' :) @@ -48,6 +177,18 @@ declare %private function zotero:resource-exists($coll as xs:string, $name as xs else some $r in xmldb:get-child-resources($coll) satisfies ($r = $name) }; +(:~ + Read the sync meta information from `$config:zotero-meta-path`. + + If the meta file does not exist or cannot be parsed, returns a default map: + { "libraryVersion": 0 } + + Return: + @return map(*) e.g. { "libraryVersion": 8306, "syncedAt": "2025-10-09T12:34:56Z" } + + Errors: + - Exceptions are caught internally; a default map is returned. +:) declare %private function zotero:read-meta() as map(*) { let $ps := zotero:path-split($config:zotero-meta-path) let $exists := zotero:resource-exists($ps?coll, $ps?name) @@ -65,7 +206,24 @@ declare %private function zotero:read-meta() as map(*) { try { parse-json($txt) } catch * { map{ "libraryVersion": 0, "syncedAt": "" } } }; -(: ─────────── overwrite meta.json; return true() on success ─────────── :) +(:~ + Write sync meta information to `$config:zotero-meta-path`. + + Overwrites or creates `meta.json` with: + { + "libraryVersion": , + "syncedAt": current-dateTime() + } + + Parameters: + @param $libraryVersion xs:integer Zotero Last-Modified-Version to persist. + + Return: + @return xs:boolean true() on success, false() on any error. + + Side-effects: + - Stores JSON with media type `application/json`. +:) declare %private function zotero:write-meta($lmv as xs:integer) as xs:boolean { let $ps := zotero:path-split($config:zotero-meta-path) let $json := serialize( @@ -92,8 +250,22 @@ declare %private function zotero:store-item($key as xs:string, $data as map(*)) ) }; +(:~ + Ingest a single Zotero page (array of items) into the local cache. + + For each array entry, extracts: + - key = item key (from top-level `key` or `data?key`) + - data = the `data` object + - bib = the `bib` string (when `include=bib` was requested) + + Stores `{ data + "bib": }` as `.json` under `$config:zotero-items-dir`. -(: Ingest one page of items including bib :) + Parameters: + @param $arr array(*) The parsed JSON array from Zotero. + + Return: + @return xs:integer Number of items successfully stored. +:) declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { let $n := array:size($arr) return @@ -129,7 +301,23 @@ declare %private function zotero:next-link($resp as element(http:response)) as x return string-join((($cands)[1]), '') (: () -> '' :) }; -(: follow pagination; $next may be empty :) +(:~ + Follow pagination and ingest subsequent pages. + + Issues an HTTP GET to `$next`, parses JSON, ingests items, and recursively + follows the next `Link: rel="next"` until exhausted. + + Parameters: + @param $next xs:string Absolute Zotero API URL taken from the Link header. + @param $acc xs:integer Accumulator of items ingested so far. + + Return: + @return xs:integer Total count of items ingested (including prior pages). + + Notes: + - Pass an empty string to stop: the function will return $acc unchanged. + - Uses the same headers as the first page (zotero:headers()). +:) declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:integer) as xs:integer { if (empty($next) or $next = '') then $acc else @@ -155,30 +343,8 @@ declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:int }; (: ───────────────────────────────────────────────────────── - Incremental sync - ───────────────────────────────────────────────────────── :) - -(: primary worker: keep your existing sync($config,$root) unchanged :) - -declare function zotero:debug-exports() as xs:string { - let $ns := "http://example.org/zotero" - let $ok2 := exists(function-lookup(QName($ns, "sync"), 2)) - let $ok1 := exists(function-lookup(QName($ns, "sync"), 1)) - let $ok0 := exists(function-lookup(QName($ns, "sync"), 0)) - return serialize(map{ - "sync#2": $ok2, "sync#1": $ok1, "sync#0": $ok0 - }, map{"method":"json","indent":true()}) -}; - - -(: ───────────────────────────────────────────────────────── - Public endpoint: POST /api/z/sync + Public endpoint: POST /api/zotero/sync ───────────────────────────────────────────────────────── :) -(: optional arity shims so Roaster can call #0/#1 too :) - -(: MAIN :) - -(: Roaster-safe wrappers; keep if your router may call #0/#1 :) declare function zotero:sync() as xs:string { response:set-header("Content-Type","application/json"), zotero:sync(map{}, ) @@ -188,9 +354,58 @@ declare function zotero:sync($request as map(*)) as xs:string { zotero:sync($request, ) }; -(: MAIN :) -(: INLINE sync — writes meta.json on BOTH 304 and 200 :) -(: ─────────── sync: ALWAYS writes meta.json (200 and 304) ─────────── :) +(:~ + Sync cached Zotero items for the configured group. + + Hits the Zotero Web API `/groups/{groupId}/items` with `include=data,bib` + (plus your CSL `style`) and `limit=100`, follows pagination via the HTTP + `Link: rel="next"` header, and writes each item as `.json` into + `$config:zotero-items-dir`. Also updates `$config:zotero-meta-path` + with the latest `libraryVersion` and `syncedAt`. + + NOTE: Returns a serialized JSON string (xs:string) intentionally to avoid + Roaster’s atomization issue with map/array results. `Content-Type` is set + to `application/json`. + + Parameters: + @param $request map(*) Roaster request context (not used here; kept for arity). + @param $root element() Roaster root element (unused). + + Return: + @return xs:string JSON string like: + { + "status":"ok|error", + "updated": , (: number of items written this call :) + "libraryVersion": , (: Zotero Last-Modified-Version :) + "metaWriteOk": true|false, (: meta.json write result :) + "metaPath": "/meta.json", + "httpStatus": , (: only on error :) + "errorBody": "", (: only on error :) + "requestHref": "" (: on error, sometimes on ok :) + } + + Headers: + - Sets `Content-Type: application/json`. + - Sends `If-Modified-Since-Version` when a previous `libraryVersion` exists. + - Adds `Zotero-API-Key` if configured in your module. + + Side-effects: + - Writes/overwrites `/.json` (merged { data + bib }). + - Writes/overwrites `meta.json` with { libraryVersion, syncedAt }. + + Status codes: + - 200 (body "status":"ok") on success or 304-from-Zotero. + - 200 (body "status":"error") with details on upstream HTTP errors. + + Example (curl): + curl -sS '…/api/zotero/sync' + + See also: + - zotero:headers() + - zotero:ingest-page() + - zotero:write-meta() + - zotero:sync-follow() +:) declare function zotero:sync($request as map(*), $root as element()) { response:set-header("Content-Type","application/json"), @@ -287,7 +502,10 @@ declare function zotero:sync($request as map(*), $root as element()) { return serialize($payload, map{ "method":"json", "indent": true() }) }; -(: ─── Wrappers so Roaster can resolve any arity ─── :) +(: ───────────────────────────────────────────────────────── + Public endpoint: POST /api/zotero/items/search + ───────────────────────────────────────────────────────── :) + declare function zotero:items-search() as xs:string { zotero:items-search(map{}, ) }; @@ -296,7 +514,49 @@ declare function zotero:items-search($request as map(*)) as xs:string { zotero:items-search($request, ) }; -(: ─── MAIN: GET /api/zotero/items/search ─── :) +(:~ + Search cached items (JSON only, from local cache). + + Scans `$config:zotero-items-dir` for `*.json`, applies optional + full-text query `q` over title/creators/DOI, optional `tag` filter, + applies `limit`, and returns a result object with counts and items. + + NOTE: Returns a serialized JSON string (xs:string). Sets + `Content-Type: application/json` and `X-Total-Count`. + + Query parameters: + - q (string) Full-text across title + creators + DOI (case-insensitive). + - tag (string) Match items that have this Zotero tag (case-insensitive). + - limit (integer) Maximum items to return (default 15; minimum 1). + + Parameters: + @param $request map(*) Roaster request context (unused beyond reading query params). + @param $root element() Roaster root element (unused). + + Return: + @return xs:string JSON like: + { + "query": { "q":"…", "tag":"…", "limit": 15 }, + "total": , (: matches before limit :) + "returned": , (: items included :) + "items": [ { "key":"…", "data": {…} }, … ] + } + + Matching rules: + - `q` is matched with `contains()` against a normalized string built from: + - title (data?title) + - creators (joined first/last/name) + - DOI (data?DOI or data?doi) + - `tag` matches if the item has ANY tag equal to the parameter (lowercased). + + Headers: + - `Content-Type: application/json` + - `X-Total-Count: ` + + Status codes: + - 200 (always) — empty result set when nothing matches. + +:) declare function zotero:items-search($request as map(*), $root as element()) as xs:string { response:set-header("Content-Type", "application/json"), @@ -371,3 +631,125 @@ declare function zotero:items-search($request as map(*), $root as element()) as return serialize($payload, map{ "method": "json", "indent": true() }) }; +(: ───────────────────────────────────────────────────────── + Public endpoint: POST /api/zotero/items/bib + ───────────────────────────────────────────────────────── :) + +(: ── wrappers (streaming: return empty-sequence()) ── :) +declare function zotero:item-bib() as empty-sequence() { + zotero:item-bib(map{}, ) +}; + +declare function zotero:item-bib($request as map(*)) as empty-sequence() { + zotero:item-bib($request, ) +}; + +(: helper: serialize a small HTML fallback safely :) +declare %private function zotero:_fallback-html($title as xs:string) as xs:string { + serialize( + { $title }, + map{ "method":"html", "omit-xml-declaration": true(), "indent": false() } + ) +}; + +(: helper: load one cached item by key (returns parsed map or empty) :) +declare %private function zotero:_load-item-by-key($key as xs:string) as item()? { + let $coll := $config:zotero-items-dir + let $uri := concat($coll, "/", $key, ".json") + let $bin := try { util:binary-doc($uri) } catch * { () } + let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" + return if ($txt ne "") then try { parse-json($txt) } catch * { () } else () +}; + +(: ── MAIN: GET /api/zotero/items/top/bib?key=... | ?tag=... ── :) +declare function zotero:item-bib($request as map(*), $root as element()) as empty-sequence() { + response:set-header("Content-Type", "text/html; charset=UTF-8"), + + let $coll := $config:zotero-items-dir + let $key := normalize-space(request:get-parameter("key", "")) + let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) + + let $emit := + function($html as xs:string) as empty-sequence() { + response:stream-binary(util:string-to-binary($html), "text/html", ()), + () + } + + return + if (not(xmldb:collection-available($coll))) then ( + response:set-status-code(404), + $emit("") + ) + + else if ($key ne "") then + let $item := zotero:_load-item-by-key($key) + return + if (empty($item)) then ( + response:set-status-code(404), + $emit("") + ) else + let $bib := string(($item?bib)[1]) + return + if (normalize-space($bib) ne "") then + $emit($bib) + else + let $parentKey := string(($item?parentItem)[1]) + return + if (normalize-space($parentKey) ne "") then + let $parent := zotero:_load-item-by-key($parentKey) + return + if (empty($parent)) then + $emit(zotero:_fallback-html(string((($item?title, $item?note)[1])))) + else + let $pb := string(($parent?bib)[1]) + return + if (normalize-space($pb) ne "") then + $emit($pb) + else + $emit(zotero:_fallback-html(string((($parent?title, $item?title, $item?note)[1])))) + else + $emit(zotero:_fallback-html(string((($item?title, $item?note)[1])))) + + else if ($tag ne "") then + let $names := xmldb:get-child-resources($coll)[ends-with(., ".json")] + let $found := + ( + for $name in $names + let $uri := concat($coll, "/", $name) + let $bin := try { util:binary-doc($uri) } catch * { () } + let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" + where $txt ne "" + let $data := try { parse-json($txt) } catch * { () } + + (: top-level only :) + let $parent := string(($data?parentItem)[1]) + where normalize-space($parent) = "" + + (: tag match, case-insensitive :) + let $tags := + if ($data?tags instance of array(*)) then + for $i in 1 to array:size($data?tags) + return lower-case(normalize-space(string((array:get($data?tags, $i)?tag)[1]))) + else () + where some $t in $tags satisfies $t = $tag + + return $data + )[1] + return + if (empty($found)) then ( + response:set-status-code(404), + $emit("") + ) + else + let $bib := string(($found?bib)[1]) + return + if (normalize-space($bib) ne "") then + $emit($bib) + else + $emit(zotero:_fallback-html(string(($found?title)[1]))) + + else ( + response:set-status-code(400), + $emit("") + ) +}; diff --git a/src/repo.xml b/src/repo.xml index 18ff9e3..d2f8b32 100644 --- a/src/repo.xml +++ b/src/repo.xml @@ -8,6 +8,6 @@ pre-install.xql post-install.xql edep - + 2022-04-19T13:07:52.079+02:00 \ No newline at end of file diff --git a/src/templates/zotero.html b/src/templates/zotero.html new file mode 100644 index 0000000..a9d0b29 --- /dev/null +++ b/src/templates/zotero.html @@ -0,0 +1,200 @@ + + + + + + + zotero + + + + + + + + +
+ + + + + + + + + + + + [{}] + + + + + + + + + + + + false + + + + + + + + + + + + loading items + + +

Zotero Test Page

+
+
+ +

Synchronize from Zotero

+ + + +

Search

+
+ + + + + open + + + busy + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +

Get a bib

+ + + + +
+ +
+ + + + + + + + +
+ +
+ + From 13137808a644d5a7a29efed8df36cc8148946b4a Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Sat, 11 Oct 2025 18:14:48 +0200 Subject: [PATCH 087/254] autosuggest component, suggest endpoint --- src/modules/custom-api.json | 19 + src/modules/lib/api/zotero.xql | 104 +++++ src/resources/scripts/zotero-autocomplete.js | 385 +++++++++++++++++++ src/templates/zotero-autocomplete.html | 36 ++ 4 files changed, 544 insertions(+) create mode 100644 src/resources/scripts/zotero-autocomplete.js create mode 100644 src/templates/zotero-autocomplete.html diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index 2d901d6..cf775b9 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -886,6 +886,25 @@ } } }, + "/api/zotero/items/suggest": { + "get": { + "summary": "Lightweight suggestions for autocomplete (key, tag, label HTML).", + "tags": ["zotero"], + "operationId": "zotero:items-suggest", + "parameters": [ + { "in": "query", "name": "q", "schema": { "type": "string" } }, + { "in": "query", "name": "tag", "schema": { "type": "string" } }, + { "in": "query", "name": "limit", "schema": { "type": "integer", "default": 8 } }, + { "in": "query", "name": "top", "schema": { "type": "boolean", "default": true } } + ], + "responses": { + "200": { + "description": "Suggestion list", + "content": { "text/plain": { "schema": { "type": "object" } } } + } + } + } + }, "/api/zotero/items/search": { "get": { "summary": "Search cached items. JSON only.", diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index b9c8f90..c26adbb 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -753,3 +753,107 @@ declare function zotero:item-bib($request as map(*), $root as element()) as empt $emit("") ) }; + +(: ── wrappers ── :) +declare function zotero:items-suggest() as xs:string { + zotero:items-suggest(map{}, ) +}; + +declare function zotero:items-suggest($request as map(*)) as xs:string { + zotero:items-suggest($request, ) +}; + +(: serialize a tiny HTML safely :) +declare %private function zotero:_as-html($n as node()) as xs:string { + serialize($n, map { "method":"html", "omit-xml-declaration": true(), "indent": false() }) +}; + +(: MAIN: GET /api/zotero/items/suggest?q=&tag=&limit=&top=1 :) +declare function zotero:items-suggest($request as map(*), $root as element()) as xs:string { + response:set-header("Content-Type", "application/json"), + + let $coll := $config:zotero-items-dir + let $qIn := lower-case(normalize-space(request:get-parameter("q", ""))) + let $tagIn := lower-case(normalize-space(request:get-parameter("tag", ""))) + let $limIn := request:get-parameter("limit", "8") + let $limit := let $n := try { xs:integer($limIn) } catch * { 8 } + return if ($n lt 1) then 8 else $n + let $topIn := request:get-parameter("top", "1") + let $topOnly:= not($topIn = ("0","false","no")) + + let $names := + if (xmldb:collection-available($coll)) + then xmldb:get-child-resources($coll)[ends-with(., ".json")] + else () + + let $matches := + for $name in $names + let $uri := concat($coll, "/", $name) + let $bin := try { util:binary-doc($uri) } catch * { () } + let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" + where $txt ne "" + let $data := try { parse-json($txt) } catch * { map{} } + + (: skip non-top-level if requested :) + let $parent := string(($data?parentItem)[1]) + where (not($topOnly)) or (normalize-space($parent) = "") + + (: quick tag set :) + let $tags := + if ($data?tags instance of array(*)) then + for $i in 1 to array:size($data?tags) + return lower-case(normalize-space(string((array:get($data?tags, $i)?tag)[1]))) + else () + let $primaryTag := string(($tags[1], "")[1]) + + (: tag filter :) + where ($tagIn = "") or (some $t in $tags satisfies $t = $tagIn) + + (: build haystack for q over title/creators/DOI :) + let $title := lower-case(string(($data?title)[1])) + let $creators := + if ($data?creators instance of array(*)) then + string-join( + for $i in 1 to array:size($data?creators) + let $c := array:get($data?creators, $i) + let $ln := string(($c?lastName)[1]) + let $fn := string(($c?firstName)[1]) + let $nm := string(($c?name)[1]) + let $parts := ($ln, $fn, $nm) + let $nonEmpty := for $p in $parts where normalize-space($p) ne "" return $p + let $one := normalize-space(string-join($nonEmpty, " ")) + where $one ne "" + return $one + , " ") + else "" + let $doi := lower-case(string((($data?DOI, $data?doi)[1]))) + let $hay := normalize-space(string-join(($title, $creators, $doi), " ")) + + where ($qIn = "") or contains($hay, $qIn) + + (: display label: cached bib or title fallback :) + let $bib := string(($data?bib)[1]) + let $label := + if (normalize-space($bib) ne "") then $bib + else zotero:_as-html({ $title }) + + let $key := string(($data?key, replace($name, "\.json$", ""))[1]) + + return map{ + "key": $key, + "tag": $primaryTag, + "label": $label + } + + let $total := count($matches) + let $limited := subsequence($matches, 1, $limit) + + let $payload := map{ + "query": map{ "q": $qIn, "tag": $tagIn, "limit": $limit, "top": $topOnly }, + "total": $total, + "returned": count($limited), + "items": array { $limited } + } + + return serialize($payload, map{ "method":"json", "indent": true() }) +}; diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js new file mode 100644 index 0000000..b050611 --- /dev/null +++ b/src/resources/scripts/zotero-autocomplete.js @@ -0,0 +1,385 @@ +/* ====================== ZOTERO AUTOCOMPLETE WEB COMPONENT ====================== */ +class ZoteroAutocomplete extends HTMLElement { + static get observedAttributes() { + return ['endpoint', 'bib-endpoint', 'tag', 'limit', 'minlength', 'debounce']; + } + + constructor() { + super(); + // state + this._items = []; + this._active = -1; + this._selected = null; + this._debounceMs = 250; + this._minlen = 2; + this._uidBase = this._uid(); + + // markup (light DOM) + this.classList.add('za'); + if (!this.querySelector('input')) { + this.innerHTML = ` +
+ + +
+ + `; + } + + this.$input = this.querySelector('.za-input') || this.querySelector('input'); + this.$clear = this.querySelector('.za-clear'); + this.$list = this.querySelector('.za-list'); + + // bind handlers + this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); + this._onKeyInput = this._handleKeyOnInput.bind(this); + this._onKeyItem = this._handleKeyOnItem.bind(this); + this._onClick = this._handleClick.bind(this); + this._onBlur = this._handleBlur.bind(this); + this._onFocus = this._handleFocus.bind(this); + this._onClear = this._handleClear.bind(this); + } + + connectedCallback() { + // configuration + this._endpoint = this.getAttribute('endpoint') || '/api/zotero/items/suggest'; + this._bibEndpoint = this.getAttribute('bib-endpoint') || '/api/zotero/items/bib'; + this._tag = this.getAttribute('tag') || ''; + this._limit = parseInt(this.getAttribute('limit') || '8', 10); + this._minlen = parseInt(this.getAttribute('minlength') || String(this._minlen), 10); + this._debounceMs = parseInt(this.getAttribute('debounce') || String(this._debounceMs), 10); + + // rebind debounce with current ms + this.$input.removeEventListener('input', this._onInput); + this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); + + // events + this.$input.addEventListener('input', this._onInput); + this.$input.addEventListener('keydown', this._onKeyInput); + this.$list.addEventListener('mousedown', this._onClick); // mousedown avoids blur before click + this.$list.addEventListener('keydown', this._onKeyItem); + this.addEventListener('focusout', this._onBlur); + this.addEventListener('focusin', this._onFocus); + this.$clear.addEventListener('click', this._onClear); + + // default look (easily overridden by page CSS) + const baseFont = '16px/1.35 system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif'; + this.$input.style.cssText = [ + `font:${baseFont}`, + 'padding:.75rem 2rem .75rem 1rem', + 'border:1px solid #ccc', + 'border-radius:.75rem', + 'width:100%', + 'box-sizing:border-box', + 'outline:none', + 'transition:border-color .15s ease', + 'background:#fff', + ].join(';'); + this.$input.addEventListener('focus', () => (this.$input.style.borderColor = '#888')); + this.$input.addEventListener('blur', () => (this.$input.style.borderColor = '#ccc')); + } + + disconnectedCallback() { + this.$input.removeEventListener('input', this._onInput); + this.$input.removeEventListener('keydown', this._onKeyInput); + this.$list.removeEventListener('mousedown', this._onClick); + this.$list.removeEventListener('keydown', this._onKeyItem); + this.removeEventListener('focusout', this._onBlur); + this.removeEventListener('focusin', this._onFocus); + this.$clear.removeEventListener('click', this._onClear); + } + + attributeChangedCallback(name, _old, value) { + if (!this.isConnected) return; + if (name === 'endpoint') this._endpoint = value || this._endpoint; + if (name === 'bib-endpoint') this._bibEndpoint = value || this._bibEndpoint; + if (name === 'tag') this._tag = value || ''; + if (name === 'limit') this._limit = parseInt(value || '8', 10); + if (name === 'minlength') this._minlen = parseInt(value || '2', 10); + if (name === 'debounce') { + this._debounceMs = parseInt(value || '250', 10); + this.$input?.removeEventListener('input', this._onInput); + this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); + this.$input?.addEventListener('input', this._onInput); + } + } + + /* ---------------------------- public API ---------------------------- */ + get value() { + return this._selected?.key || ''; + } + get selected() { + return this._selected || null; + } + clear() { + this.$input.value = ''; + this.$input.dataset.key = ''; + this._selected = null; + this._render([]); + this._toggleClear(); + } + + /* ---------------------------- internals ---------------------------- */ + async _handleInput(e) { + const q = e.target.value.trim(); + this._selected = null; + this._toggleClear(); + if (q.length < this._minlen) return this._render([]); + + try { + const url = new URL(this._endpoint, window.location.href); + url.searchParams.set('q', q); + if (this._tag) url.searchParams.set('tag', this._tag); + if (this._limit) url.searchParams.set('limit', String(this._limit)); + const res = await fetch(url.toString(), { credentials: 'include' }); + if (!res.ok) throw new Error('HTTP ' + res.status); + let data = await res.json(); + const items = Array.isArray(data) ? data : data.items || []; + // Normalize to { key, title, bib? } + let list = items + .map(it => ({ + key: it.key || it.data?.key || '', + title: it.title || it.data?.title || '', + bib: it.bib || it.html || '', + })) + .filter(x => x.key); + + // If endpoint didn’t include bib/html, fetch per-item (limited) + if (list.length && !list[0].bib) { + const limited = list.slice(0, this._limit); + const htmls = await Promise.all(limited.map(i => this._fetchBib(i.key).catch(() => ''))); + limited.forEach((i, idx) => (i.bib = htmls[idx] || this._escape(i.title || '[untitled]'))); + list = limited; + } + + this._render(list); + } catch (err) { + console.error('[zotero-autocomplete] suggest error:', err); + this._render([]); + } + } + + async _fetchBib(key) { + const url = new URL(this._bibEndpoint, window.location.href); + url.searchParams.set('key', key); + const res = await fetch(url.toString(), { credentials: 'include' }); + if (!res.ok) throw new Error('HTTP ' + res.status); + return await res.text(); // HTML snippet + } + + _render(items) { + // build list + this._items = items; + this._active = -1; + this.$list.innerHTML = ''; + if (!items.length) { + this.$list.style.display = 'none'; + this.$input.setAttribute('aria-expanded', 'false'); + return; + } + + items.forEach((it, idx) => { + const id = `${this._uidBase}-opt-${idx}`; + const li = document.createElement('li'); + li.className = 'za-item'; + li.id = id; + li.setAttribute('role', 'option'); + li.setAttribute('data-idx', String(idx)); + li.setAttribute('data-key', it.key); + li.tabIndex = -1; // focusable via JS/Tab sequence + li.style.cssText = 'padding:.5rem .75rem;border-radius:.5rem;margin:.15rem 0;cursor:pointer;outline:none;'; + // bib is HTML from our API; fallback is escaped title + li.innerHTML = ` +
+ ${it.bib || this._escape(it.title || '[untitled]')} +
+ `; + // Mouse enter highlights + li.addEventListener('mouseenter', () => this._setActive(idx, true /*noFocus*/)); + this.$list.appendChild(li); + }); + + this.$list.style.display = 'block'; + this.$input.setAttribute('aria-expanded', 'true'); + this.$input.setAttribute('aria-activedescendant', ''); + this._toggleClear(); + } + + /* ------------- keyboard handling ------------- */ + _handleKeyOnInput(e) { + const hasMenu = this._items.length > 0; + if (!hasMenu && e.key === 'Tab') return; // nothing to move to + + switch (e.key) { + case 'ArrowDown': + if (hasMenu) { + e.preventDefault(); + this._focusItem(0); + } + break; + case 'Tab': + if (hasMenu && !e.shiftKey) { + // Move focus into the list (first item) + e.preventDefault(); + this._focusItem(0); + } + break; + case 'Escape': + this._render([]); + break; + default: + // no-op + break; + } + } + + _handleKeyOnItem(e) { + const li = e.target.closest('.za-item'); + if (!li) return; + const idx = parseInt(li.getAttribute('data-idx') || '-1', 10); + if (idx < 0) return; + const last = this._items.length - 1; + + switch (e.key) { + case 'ArrowDown': + e.preventDefault(); + this._focusItem(Math.min(last, idx + 1)); + break; + case 'ArrowUp': + e.preventDefault(); + if (idx === 0) { + // back to input + this.$input.focus(); + this._setActive(-1, true); + } else { + this._focusItem(idx - 1); + } + break; + case 'Enter': + case ' ': + e.preventDefault(); + this._choose(idx); + break; + case 'Tab': + if (!e.shiftKey) { + // select on Tab forward + e.preventDefault(); + this._choose(idx); + } // Shift+Tab: allow bubbling to move focus back out + break; + case 'Escape': + this._render([]); + this.$input.focus(); + break; + default: + break; + } + } + + /* ------------- mouse handling ------------- */ + _handleClick(e) { + const li = e.target.closest('.za-item'); + if (!li) return; + const idx = parseInt(li.getAttribute('data-idx') || '-1', 10); + if (idx >= 0) this._choose(idx); + } + + /* ------------- focus/blur ------------- */ + _handleBlur(e) { + const related = e.relatedTarget; + if (!this.contains(related)) { + this._render([]); // hide list when focus leaves the component + } + } + _handleFocus() { + const q = this.$input.value.trim(); + if (q.length >= this._minlen && this._items.length) { + this.$list.style.display = 'block'; + this.$input.setAttribute('aria-expanded', 'true'); + } + this._toggleClear(); + } + + /* ------------- clear button ------------- */ + _handleClear() { + this.clear(); + this.$input.focus(); + } + _toggleClear() { + const show = !!(this.$input.value || this._selected); + this.$clear.style.display = show ? 'block' : 'none'; + } + + /* ------------- helpers ------------- */ + _setActive(idx, noFocus = false) { + const items = Array.from(this.$list.children); + items.forEach(el => { + el.style.background = ''; + el.setAttribute('aria-selected', 'false'); + }); + this._active = idx; + if (idx >= 0 && items[idx]) { + items[idx].style.background = 'rgba(0,0,0,.06)'; + items[idx].setAttribute('aria-selected', 'true'); + this.$input.setAttribute('aria-activedescendant', items[idx].id || ''); + if (!noFocus) items[idx].focus({ preventScroll: false }); + items[idx].scrollIntoView({ block: 'nearest' }); + } else { + this.$input.setAttribute('aria-activedescendant', ''); + } + } + _focusItem(idx) { + this._setActive(idx); + } + + _choose(idx) { + const item = this._items[idx]; + if (!item) return; + this._selected = item; + // Put a human-friendly value in the input; keep key in dataset + this.$input.value = this._stripHtml(item.bib) || item.title || ''; + this.$input.dataset.key = item.key; + this._render([]); // hide + this._toggleClear(); // show the clear button + + this.dispatchEvent( + new CustomEvent('zotero-select', { + bubbles: true, + detail: { key: item.key, title: item.title || '', bib: item.bib || '' }, + }), + ); + } + + _debounce(fn, ms) { + let t = null; + return (...args) => { + clearTimeout(t); + t = setTimeout(() => fn.apply(this, args), ms); + }; + } + _escape(s) { + return String(s).replace( + /[&<>"']/g, + ch => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[ch], + ); + } + _stripHtml(s) { + const tmp = document.createElement('div'); + tmp.innerHTML = s || ''; + return tmp.textContent || ''; + } + _uid() { + return Math.random().toString(36).slice(2); + } +} + +customElements.define('zotero-autocomplete', ZoteroAutocomplete); +/* ==================== /ZOTERO AUTOCOMPLETE WEB COMPONENT ==================== */ diff --git a/src/templates/zotero-autocomplete.html b/src/templates/zotero-autocomplete.html new file mode 100644 index 0000000..d4a4401 --- /dev/null +++ b/src/templates/zotero-autocomplete.html @@ -0,0 +1,36 @@ + + + + + + + zotero + + + + + + + + +
+ + +
+ + + + From 7117cf1f8df4991098abd84fc23c60100f69c8e2 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Mon, 13 Oct 2025 14:15:31 +0200 Subject: [PATCH 088/254] new matching rule in controller to make sure we get proper doctype html returned (probably generify for other files to avoid quirks mode) --- src/controller.xql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/controller.xql b/src/controller.xql index cd62ecd..2643d79 100644 --- a/src/controller.xql +++ b/src/controller.xql @@ -32,7 +32,13 @@ if ($exist:path eq '' or matches($exist:path, "^/edit/[^/]+$")) then - +else if (matches($exist:path, "^/?templates/zotero-autocomplete\.html$")) then ( + util:declare-option( + "exist:serialize", + "method=html5 media-type=text/html omit-xml-declaration=yes" + ), + doc('templates/zotero-autocomplete.html') (: no stream-binary; just return the node :) +) else if ($exist:path eq "/") then (: forward root path to index.xql :) From 560a580d2b8ea9f1c7bd994c1f2246abd25fa750 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Mon, 13 Oct 2025 14:22:39 +0200 Subject: [PATCH 089/254] new zotero-autocomplete component inkl. testpage with Fore binding --- src/resources/css/zotero-autocomplete.css | 108 ++++++++ src/resources/scripts/zotero-autocomplete.js | 264 ++++++++++++------- src/templates/zotero-autocomplete.html | 62 +++-- 3 files changed, 310 insertions(+), 124 deletions(-) create mode 100644 src/resources/css/zotero-autocomplete.css diff --git a/src/resources/css/zotero-autocomplete.css b/src/resources/css/zotero-autocomplete.css new file mode 100644 index 0000000..4a360f3 --- /dev/null +++ b/src/resources/css/zotero-autocomplete.css @@ -0,0 +1,108 @@ +/* --- sizing tokens --- */ +zotero-autocomplete .za-field{ + --za-clear-w: 2rem; + --za-pad-x: 1rem; + --za-right-pad: calc(var(--za-clear-w) + var(--za-pad-x)); + + /* Stack overlay + input */ + display: grid; + grid-template-areas: "stack"; + position: relative; + + /* Move border/background to the wrapper so it can grow with overlay */ + padding: .75rem var(--za-right-pad) .75rem var(--za-pad-x); + border: 1px solid #ccc; + border-radius: .75rem; + background: #fff; + transition: border-color .15s ease, box-shadow .15s ease; +} + +/* Focus ring now on the wrapper (covers the entire wrapped overlay) */ +zotero-autocomplete .za-field:focus-within{ + border-color: #4c9ffe; + box-shadow: 0 0 0 3px rgba(76,159,254,.2); +} + +/* Input: borderless, transparent, stretched to wrapper height */ +zotero-autocomplete .za-input{ + grid-area: stack; + background: transparent; + border: 0; + padding: 0; /* padding lives on the wrapper */ + margin: 0; + width: 100%; + height: 100%; /* << stretch with overlay height */ + min-height: 1.35em; /* a line at least */ + outline: none; + font: 16px/1.35 system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; + color: inherit; +} + +/* Overlay: multiline; sits above input; no inner padding now */ +zotero-autocomplete .za-overlay{ + grid-area: stack; + white-space: normal; + word-break: break-word; + overflow: hidden; + display: none; + color: #111; + font: 14px/1.45 system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; + pointer-events: none; + z-index: 1; +} +zotero-autocomplete .za-field.has-overlay .za-overlay{ display:block; } + +/* Hide input text paint when overlay is shown; caret stays visible */ +zotero-autocomplete .za-field.has-overlay .za-input{ + color: transparent; -webkit-text-fill-color: transparent; + caret-color: #111; +} + +/* Clear button reserved space & always clickable */ +zotero-autocomplete .za-clear{ + position: absolute; right: .5rem; top: 50%; transform: translateY(-50%); + border: 0; background: transparent; cursor: pointer; + font-size: 1.2rem; line-height: 1; color: #888; display: none; + z-index: 3; +} +zotero-autocomplete .za-clear.is-visible{ display:block; } + +/* Suggestions list above */ +zotero-autocomplete .za-list{ + list-style:none; margin:.25rem 0 0; padding:.25rem; + border:1px solid #ddd; border-radius:.5rem; background:#fff; + box-shadow:0 4px 14px rgba(0,0,0,.08); + max-height:320px; overflow:auto; display:none; position:relative; z-index: 5; +} +zotero-autocomplete .za-list.is-open{ display:block; } + +zotero-autocomplete .za-item{ + padding:.5rem .75rem; border-radius:.5rem; margin:.15rem 0; cursor:pointer; outline:none; +} +zotero-autocomplete .za-item[aria-selected="true"], +zotero-autocomplete .za-item:hover{ background:rgba(0,0,0,.06); } + +zotero-autocomplete .za-bib{ + white-space: normal; word-break: break-word; + font:14px/1.45 system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; +} +/* when overlay is shown, prevent selection highlight on the input */ +zotero-autocomplete .za-field.has-overlay .za-input { + user-select: none; /* no text selection */ + -webkit-user-select: none; +} + +/* just in case something still gets selected, make the highlight transparent */ +zotero-autocomplete .za-field.has-overlay .za-input::selection { + background: transparent; +} +zotero-autocomplete .za-field.has-overlay .za-input::-moz-selection { + background: transparent; +} + +/* also remove any tap/focus tint some browsers draw */ +zotero-autocomplete .za-input { + -webkit-tap-highlight-color: transparent; + outline: none; + box-shadow: none; +} diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index b050611..3dff499 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -1,7 +1,8 @@ -/* ====================== ZOTERO AUTOCOMPLETE WEB COMPONENT ====================== */ +// resources/scripts/zotero-autocomplete.js +// Light-DOM web component with Fore-friendly events & multiline overlay class ZoteroAutocomplete extends HTMLElement { static get observedAttributes() { - return ['endpoint', 'bib-endpoint', 'tag', 'limit', 'minlength', 'debounce']; + return ['endpoint', 'bib-endpoint', 'tag', 'limit', 'minlength', 'debounce', 'value', 'name']; } constructor() { @@ -12,33 +13,31 @@ class ZoteroAutocomplete extends HTMLElement { this._selected = null; this._debounceMs = 250; this._minlen = 2; - this._uidBase = this._uid(); + this._uidBase = Math.random().toString(36).slice(2); - // markup (light DOM) + // light DOM markup this.classList.add('za'); if (!this.querySelector('input')) { this.innerHTML = ` -
+
- + placeholder="Search references…"> + +
- +
    `; } + // refs + this.$field = this.querySelector('.za-field'); this.$input = this.querySelector('.za-input') || this.querySelector('input'); + this.$overlay = this.querySelector('.za-overlay'); this.$clear = this.querySelector('.za-clear'); this.$list = this.querySelector('.za-list'); - // bind handlers + // handlers this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); this._onKeyInput = this._handleKeyOnInput.bind(this); this._onKeyItem = this._handleKeyOnItem.bind(this); @@ -49,7 +48,7 @@ class ZoteroAutocomplete extends HTMLElement { } connectedCallback() { - // configuration + // config this._endpoint = this.getAttribute('endpoint') || '/api/zotero/items/suggest'; this._bibEndpoint = this.getAttribute('bib-endpoint') || '/api/zotero/items/bib'; this._tag = this.getAttribute('tag') || ''; @@ -57,34 +56,25 @@ class ZoteroAutocomplete extends HTMLElement { this._minlen = parseInt(this.getAttribute('minlength') || String(this._minlen), 10); this._debounceMs = parseInt(this.getAttribute('debounce') || String(this._debounceMs), 10); - // rebind debounce with current ms - this.$input.removeEventListener('input', this._onInput); - this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); - - // events + // listeners this.$input.addEventListener('input', this._onInput); this.$input.addEventListener('keydown', this._onKeyInput); - this.$list.addEventListener('mousedown', this._onClick); // mousedown avoids blur before click + this.$list.addEventListener('mousedown', this._onClick); this.$list.addEventListener('keydown', this._onKeyItem); this.addEventListener('focusout', this._onBlur); this.addEventListener('focusin', this._onFocus); this.$clear.addEventListener('click', this._onClear); - // default look (easily overridden by page CSS) - const baseFont = '16px/1.35 system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif'; - this.$input.style.cssText = [ - `font:${baseFont}`, - 'padding:.75rem 2rem .75rem 1rem', - 'border:1px solid #ccc', - 'border-radius:.75rem', - 'width:100%', - 'box-sizing:border-box', - 'outline:none', - 'transition:border-color .15s ease', - 'background:#fff', - ].join(';'); - this.$input.addEventListener('focus', () => (this.$input.style.borderColor = '#888')); - this.$input.addEventListener('blur', () => (this.$input.style.borderColor = '#ccc')); + // optional name passthrough + const nameAttr = this.getAttribute('name'); + if (nameAttr) this.$input.name = nameAttr; + + // ensure at least one notification after connect + queueMicrotask(() => this._notifyValueChanged()); + + // preset value via attribute (fires notifications as well) + const initVal = this.getAttribute('value'); + if (initVal) this.value = initVal; } disconnectedCallback() { @@ -110,12 +100,68 @@ class ZoteroAutocomplete extends HTMLElement { this._onInput = this._debounce(this._handleInput.bind(this), this._debounceMs); this.$input?.addEventListener('input', this._onInput); } + if (name === 'name' && value) this.$input.name = value; + if (name === 'value' && value !== this.value) this.value = value || ''; } - /* ---------------------------- public API ---------------------------- */ + /* ===== Fore contract ===== */ get value() { - return this._selected?.key || ''; + return this.$input?.dataset.key || ''; } + set value(v) { + const key = String(v || '').trim(); + if (!key) { + this.clear(); + this._notifyValueChanged(); + return; + } + + this.$input.dataset.key = key; + this._selected = { key, title: '', bib: '' }; + this._fetchBib(key) + .then(html => { + this._selected.bib = html || ''; + const plain = this._stripHtml(html || '') || ''; + this.$input.value = plain; + this._showOverlay(html || plain); + this._toggleClear(); + this._notifyValueChanged(); + }) + .catch(() => { + this.$input.value = key; + this._showOverlay(key); + this._toggleClear(); + this._notifyValueChanged(); + }); + } + setValue(v) { + this.value = v; + } + getValue() { + return this.value; + } + + _notifyValueChanged() { + const val = this.value; + if (val) this.setAttribute('value', val); + else this.removeAttribute('value'); + + const opts = { bubbles: true, composed: true }; + const withDetail = name => new CustomEvent(name, { ...opts, detail: { value: val } }); + + // fire on host + this.dispatchEvent(new Event('input', opts)); + this.dispatchEvent(new Event('change', opts)); + this.dispatchEvent(withDetail('value-changed')); + // and on inner input (compat) + if (this.$input) { + this.$input.dispatchEvent(new Event('input', opts)); + this.$input.dispatchEvent(new Event('change', opts)); + this.$input.dispatchEvent(withDetail('value-changed')); + } + } + + /* ===== public helpers ===== */ get selected() { return this._selected || null; } @@ -124,15 +170,30 @@ class ZoteroAutocomplete extends HTMLElement { this.$input.dataset.key = ''; this._selected = null; this._render([]); + this._hideOverlay(); this._toggleClear(); } - /* ---------------------------- internals ---------------------------- */ + /* ===== input/search ===== */ async _handleInput(e) { + // ignore synthetic input events (Fore/programmatic) to prevent overlay flicker + if (e && e.isTrusted === false) return; + const q = e.target.value.trim(); - this._selected = null; + + // real typing → drop selection & overlay and notify empty value + if (this._selected) { + this._selected = null; + this.$input.dataset.key = ''; + this._hideOverlay(); + this._notifyValueChanged(); + } this._toggleClear(); - if (q.length < this._minlen) return this._render([]); + + if (q.length < this._minlen) { + this._render([]); + return; + } try { const url = new URL(this._endpoint, window.location.href); @@ -141,10 +202,8 @@ class ZoteroAutocomplete extends HTMLElement { if (this._limit) url.searchParams.set('limit', String(this._limit)); const res = await fetch(url.toString(), { credentials: 'include' }); if (!res.ok) throw new Error('HTTP ' + res.status); - let data = await res.json(); - const items = Array.isArray(data) ? data : data.items || []; - // Normalize to { key, title, bib? } - let list = items + const data = await res.json(); + let list = (Array.isArray(data) ? data : data.items || []) .map(it => ({ key: it.key || it.data?.key || '', title: it.title || it.data?.title || '', @@ -152,14 +211,13 @@ class ZoteroAutocomplete extends HTMLElement { })) .filter(x => x.key); - // If endpoint didn’t include bib/html, fetch per-item (limited) + // If no bib included, fetch snippets for visible set if (list.length && !list[0].bib) { const limited = list.slice(0, this._limit); const htmls = await Promise.all(limited.map(i => this._fetchBib(i.key).catch(() => ''))); limited.forEach((i, idx) => (i.bib = htmls[idx] || this._escape(i.title || '[untitled]'))); list = limited; } - this._render(list); } catch (err) { console.error('[zotero-autocomplete] suggest error:', err); @@ -172,16 +230,16 @@ class ZoteroAutocomplete extends HTMLElement { url.searchParams.set('key', key); const res = await fetch(url.toString(), { credentials: 'include' }); if (!res.ok) throw new Error('HTTP ' + res.status); - return await res.text(); // HTML snippet + return await res.text(); } + /* ===== render list ===== */ _render(items) { - // build list this._items = items; this._active = -1; this.$list.innerHTML = ''; if (!items.length) { - this.$list.style.display = 'none'; + this.$list.classList.remove('is-open'); this.$input.setAttribute('aria-expanded', 'false'); return; } @@ -194,30 +252,22 @@ class ZoteroAutocomplete extends HTMLElement { li.setAttribute('role', 'option'); li.setAttribute('data-idx', String(idx)); li.setAttribute('data-key', it.key); - li.tabIndex = -1; // focusable via JS/Tab sequence - li.style.cssText = 'padding:.5rem .75rem;border-radius:.5rem;margin:.15rem 0;cursor:pointer;outline:none;'; - // bib is HTML from our API; fallback is escaped title - li.innerHTML = ` -
    - ${it.bib || this._escape(it.title || '[untitled]')} -
    - `; - // Mouse enter highlights - li.addEventListener('mouseenter', () => this._setActive(idx, true /*noFocus*/)); + li.tabIndex = -1; + li.innerHTML = `
    ${it.bib || this._escape(it.title || '[untitled]')}
    `; + li.addEventListener('mouseenter', () => this._setActive(idx, true)); this.$list.appendChild(li); }); - this.$list.style.display = 'block'; + this.$list.classList.add('is-open'); this.$input.setAttribute('aria-expanded', 'true'); this.$input.setAttribute('aria-activedescendant', ''); this._toggleClear(); } - /* ------------- keyboard handling ------------- */ + /* ===== keyboard ===== */ _handleKeyOnInput(e) { const hasMenu = this._items.length > 0; - if (!hasMenu && e.key === 'Tab') return; // nothing to move to - + if (!hasMenu && e.key === 'Tab') return; switch (e.key) { case 'ArrowDown': if (hasMenu) { @@ -227,7 +277,6 @@ class ZoteroAutocomplete extends HTMLElement { break; case 'Tab': if (hasMenu && !e.shiftKey) { - // Move focus into the list (first item) e.preventDefault(); this._focusItem(0); } @@ -235,9 +284,6 @@ class ZoteroAutocomplete extends HTMLElement { case 'Escape': this._render([]); break; - default: - // no-op - break; } } @@ -256,12 +302,9 @@ class ZoteroAutocomplete extends HTMLElement { case 'ArrowUp': e.preventDefault(); if (idx === 0) { - // back to input this.$input.focus(); this._setActive(-1, true); - } else { - this._focusItem(idx - 1); - } + } else this._focusItem(idx - 1); break; case 'Enter': case ' ': @@ -270,21 +313,18 @@ class ZoteroAutocomplete extends HTMLElement { break; case 'Tab': if (!e.shiftKey) { - // select on Tab forward e.preventDefault(); this._choose(idx); - } // Shift+Tab: allow bubbling to move focus back out + } break; case 'Escape': this._render([]); this.$input.focus(); break; - default: - break; } } - /* ------------- mouse handling ------------- */ + /* ===== mouse ===== */ _handleClick(e) { const li = e.target.closest('.za-item'); if (!li) return; @@ -292,42 +332,63 @@ class ZoteroAutocomplete extends HTMLElement { if (idx >= 0) this._choose(idx); } - /* ------------- focus/blur ------------- */ + /* ===== focus/blur ===== */ _handleBlur(e) { const related = e.relatedTarget; - if (!this.contains(related)) { - this._render([]); // hide list when focus leaves the component - } + if (!this.contains(related)) this._render([]); } + /* _handleFocus() { + const q = this.$input.value.trim(); + if (q.length >= this._minlen && this._items.length) { + this.$list.classList.add('is-open'); + this.$input.setAttribute('aria-expanded', 'true'); + } + this._toggleClear(); + } */ _handleFocus() { + // if overlay is active, ensure no text selection band is visible + if (this.$field?.classList.contains('has-overlay')) { + const len = this.$input.value.length; + try { + this.$input.setSelectionRange(len, len); + } catch (_) {} + } + const q = this.$input.value.trim(); if (q.length >= this._minlen && this._items.length) { - this.$list.style.display = 'block'; + this.$list.classList.add('is-open'); this.$input.setAttribute('aria-expanded', 'true'); } this._toggleClear(); } - /* ------------- clear button ------------- */ + /* ===== clear ===== */ _handleClear() { this.clear(); + this._notifyValueChanged(); // Fore: value -> "" this.$input.focus(); } - _toggleClear() { - const show = !!(this.$input.value || this._selected); - this.$clear.style.display = show ? 'block' : 'none'; + + /* ===== overlay ===== */ + _showOverlay(htmlOrText) { + if (!this.$overlay) return; + this.$overlay.innerHTML = htmlOrText || ''; + this.$field?.classList.toggle('has-overlay', !!htmlOrText); + } + _hideOverlay() { + if (!this.$overlay) return; + this.$overlay.innerHTML = ''; + this.$field?.classList.remove('has-overlay'); } - /* ------------- helpers ------------- */ + /* ===== selection ===== */ _setActive(idx, noFocus = false) { const items = Array.from(this.$list.children); items.forEach(el => { - el.style.background = ''; el.setAttribute('aria-selected', 'false'); }); this._active = idx; if (idx >= 0 && items[idx]) { - items[idx].style.background = 'rgba(0,0,0,.06)'; items[idx].setAttribute('aria-selected', 'true'); this.$input.setAttribute('aria-activedescendant', items[idx].id || ''); if (!noFocus) items[idx].focus({ preventScroll: false }); @@ -344,12 +405,14 @@ class ZoteroAutocomplete extends HTMLElement { const item = this._items[idx]; if (!item) return; this._selected = item; - // Put a human-friendly value in the input; keep key in dataset - this.$input.value = this._stripHtml(item.bib) || item.title || ''; + const plain = this._stripHtml(item.bib) || item.title || ''; + this.$input.value = plain; this.$input.dataset.key = item.key; - this._render([]); // hide - this._toggleClear(); // show the clear button + this._render([]); // hide menu + this._showOverlay(item.bib || plain); + this._toggleClear(); + this._notifyValueChanged(); // Fore: emit this.dispatchEvent( new CustomEvent('zotero-select', { bubbles: true, @@ -358,11 +421,16 @@ class ZoteroAutocomplete extends HTMLElement { ); } + /* ===== utils ===== */ + _toggleClear() { + const show = !!(this.$input.value || this._selected); + this.$clear.classList.toggle('is-visible', show); + } _debounce(fn, ms) { let t = null; - return (...args) => { + return (...a) => { clearTimeout(t); - t = setTimeout(() => fn.apply(this, args), ms); + t = setTimeout(() => fn.apply(this, a), ms); }; } _escape(s) { @@ -376,10 +444,6 @@ class ZoteroAutocomplete extends HTMLElement { tmp.innerHTML = s || ''; return tmp.textContent || ''; } - _uid() { - return Math.random().toString(36).slice(2); - } } customElements.define('zotero-autocomplete', ZoteroAutocomplete); -/* ==================== /ZOTERO AUTOCOMPLETE WEB COMPONENT ==================== */ diff --git a/src/templates/zotero-autocomplete.html b/src/templates/zotero-autocomplete.html index d4a4401..7280939 100644 --- a/src/templates/zotero-autocomplete.html +++ b/src/templates/zotero-autocomplete.html @@ -1,36 +1,50 @@ - - - + zotero - - - - + + + + + + +
    - - -
    + + + + + JECUYVKL + + + - + + + + + + +

    Selected key:

    +     +
    From 44a8d4f3c30e3068eec76e3e7dd2f378b0a7242b Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Mon, 13 Oct 2025 19:32:53 +0200 Subject: [PATCH 090/254] syncing as xml also and searching on that. --- src/modules/config.xqm | 2 + src/modules/custom-api.xql | 2 +- src/modules/lib/api.xql | 2 +- src/modules/lib/api/zotero.xql | 1098 ++++++++++---------------------- src/post-install.xql | 5 + 5 files changed, 350 insertions(+), 759 deletions(-) diff --git a/src/modules/config.xqm b/src/modules/config.xqm index 6a1efa8..e631950 100644 --- a/src/modules/config.xqm +++ b/src/modules/config.xqm @@ -345,6 +345,8 @@ declare variable $config:zotero-base-dir := $config:data-root || "/zotero/groups (: Derived paths for this group :) declare variable $config:zotero-group-dir := $config:zotero-base-dir || "/" || $config:zotero-group-id; declare variable $config:zotero-items-dir := $config:zotero-group-dir || "/items"; +declare variable $config:zotero-items-xml-dir := $config:zotero-group-dir || "/items-xml"; + declare variable $config:zotero-meta-path := $config:zotero-group-dir || "/meta.json"; declare variable $config:zotero-style := "digital-humanities-im-deutschsprachigen-raum"; diff --git a/src/modules/custom-api.xql b/src/modules/custom-api.xql index 2c7d041..20f6d2b 100644 --- a/src/modules/custom-api.xql +++ b/src/modules/custom-api.xql @@ -13,7 +13,7 @@ import module namespace config="http://www.tei-c.org/tei-simple/config" at "conf import module namespace pm-config="http://www.tei-c.org/tei-simple/pm-config" at "pm-config.xql"; import module namespace tpu="http://www.tei-c.org/tei-publisher/util" at "lib/util.xql"; import module namespace errors = "http://e-editiones.org/roaster/errors"; -import module namespace zotero = "http://teipublisher.com/api/zotero" at "lib/api/zotero.xql"; +import module namespace zotero = "http://e-editiones.org/edep/api/zotero" at "lib/api/zotero.xql"; declare namespace json="http://www.json.org"; declare namespace tei="http://www.tei-c.org/ns/1.0"; diff --git a/src/modules/lib/api.xql b/src/modules/lib/api.xql index 066976e..ae427de 100644 --- a/src/modules/lib/api.xql +++ b/src/modules/lib/api.xql @@ -17,7 +17,7 @@ import module namespace vapi="http://teipublisher.com/api/view" at "api/view.xql import module namespace anno="http://teipublisher.com/api/annotations" at "api/annotations.xql"; import module namespace custom="http://teipublisher.com/api/custom" at "../custom-api.xql"; import module namespace nlp="http://teipublisher.com/api/nlp" at "api/nlp.xql"; -import module namespace zotero="http://teipublisher.com/api/zotero" at "api/zotero.xql"; +import module namespace zotero="http://e-editiones.org/edep/api/zotero" at "api/zotero.xql"; declare option output:indent "no"; diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index c26adbb..888912f 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -1,431 +1,258 @@ xquery version "3.1"; -module namespace zotero = "http://teipublisher.com/api/zotero"; +module namespace zotero = "http://e-editiones.org/edep/api/zotero"; -declare namespace map = "http://www.w3.org/2005/xpath-functions/map"; -declare namespace http = "http://expath.org/ns/http-client"; -declare namespace xmldb = "http://exist-db.org/xquery/xmldb"; -declare namespace util = "http://exist-db.org/xquery/util"; -import module namespace response = "http://exist-db.org/xquery/response"; +declare namespace http = "http://expath.org/ns/http-client"; +declare namespace request = "http://exist-db.org/xquery/request"; +declare namespace response = "http://exist-db.org/xquery/response"; +declare namespace xmldb = "http://exist-db.org/xquery/xmldb"; +declare namespace util = "http://exist-db.org/xquery/util"; import module namespace config = "http://www.tei-c.org/tei-simple/config" at "../../config.xqm"; -(:~ - ============================================================================== - Zotero cache + lookup module - ============================================================================== - - Purpose - ------- - Provide a thin caching layer for a single Zotero group and a set of read APIs - that the Fore UI can call with low latency. The module: - • syncs items from the Zotero Web API (data + bib), - • stores each record as .json in the local DB, - • exposes lightweight endpoints for search and for rendering a single - bibliography (or a safe title fallback) as HTML. - - Public endpoints (Roaster) - -------------------------- - Each endpoint exposes 3 arities so Roaster can bind it: - f(), f($request as map(*)), f($request as map(*), $root as element()). - - 1) zotero:sync($request, $root) as xs:string - - Fetches from Zotero /groups/{groupId}/items with include=data,bib and - the configured CSL style; follows Link rel="next". - - Writes items to $config:zotero-items-dir as .json. - - Updates meta.json with libraryVersion and syncedAt. - - Returns a serialized JSON string (xs:string). Sets Content-Type: application/json. - - NOTE: We serialize on purpose to avoid Roaster atomizing map/array - values (FOTY0013). If Roaster is patched later, this can return map(*). - - 2) zotero:items-search($request, $root) as xs:string - - Query params: q (full-text over title + creators + DOI), tag (exact, - case-insensitive), limit (default 15). - - Returns serialized JSON with { query, total, returned, items[] }. - - Sets headers: Content-Type: application/json, X-Total-Count: . - - 3) zotero:item-bib-top($request, $root) as xs:string - - Query params: - key = item key; OR - tag = tag name (top-level only, first match). - - Returns a single HTML snippet: - • bib string if present, else - • Title or note (safely serialized). - - Sets Content-Type: text/html; charset=UTF-8. - - Configuration (from modules/config.xqm) - --------------------------------------- - The module expects these variables to be provided by your app config: - - $config:zotero-api-base xs:string - Base URL for Zotero Web API, e.g. "https://api.zotero.org". - - $config:zotero-group-id xs:string or xs:integer - The single group to sync, e.g. "2519759". - - $config:zotero-style xs:string - CSL style id for server-side bibliography rendering, e.g. - "chicago-note-bibliography" or "digital-humanities-im-deutschsprachigen-raum". - - $config:zotero-items-dir xs:string - Collection where item JSON files are stored, e.g. - "/db/apps/edep-data/zotero/groups/2519759/items". - - $config:zotero-meta-path xs:string - Full resource path to meta.json, e.g. - "/db/apps/edep-data/zotero/groups/2519759/meta.json". - - $config:zotero-api-key xs:string (optional) - API key if the group requires auth. Public groups can omit this. - - Storage layout (created by post-install) - ---------------------------------------- - /db/apps/edep-data/zotero/ - groups/ - {groupId}/ - items/ - .json (serialized map { data: {...}, bib: "" }) - meta.json (serialized map { libraryVersion: int, syncedAt: dateTime }) - - Error handling and status - ------------------------- - • Upstream HTTP errors return a JSON body { status: "error", httpStatus, ... }. - • Sync handles 304 Not Modified and updates meta.json accordingly. - • Search always returns 200 with an empty result set when nothing matches. - • item-bib-top returns: - 200 on success, - 400 when both key and tag are missing, - 404 when the cache is missing or no item was found. - - Notes on Roaster JSON handling - ------------------------------ - Roaster 1.10.0 atomizes function results; XDM maps/arrays are function - items and cannot be atomized (FOTY0013). Until Roaster gains native JSON - output for map/array values, endpoints in this module serialize their - responses to xs:string and set the Content-Type header themselves. - In OpenAPI, you can declare "text/plain" for such responses to avoid - double-encoding; clients still see valid JSON because the handler sets - application/json. - - ============================================================================== -:) - - -declare %private function zotero:json($v as item()*) as xs:string { - serialize($v, map{"method":"json","indent":true()}) +(: --------------------------------------------------------------------------- + CONFIG (provided by your app's modules/config.xqm) + We assume these exist: + $config:zotero-api-base (e.g. "https://api.zotero.org") + $config:zotero-group-id (e.g. "2519759") + $config:zotero-style (e.g. "digital-humanities-im-deutschsprachigen-raum") + $config:zotero-meta-path (abs resource path to meta.json) + $config:zotero-items-dir (collection for JSON items) + $config:zotero-items-xml-dir (collection for XML mirror; you set '/items-xml') + $config:zotero-api-key (optional) +--------------------------------------------------------------------------- :) +(: ---- RESOLVED CONFIG (computed once) -------------------------------- :) + +(: base collection holding JSON items, as configured :) +declare variable $zotero:ITEMS_DIR as xs:string := $config:zotero-items-dir; + +(: derive the Zotero group base “…/zotero/groups/{id}” from ITEMS_DIR :) +declare variable $zotero:GROUP_BASE as xs:string := + substring-before($zotero:ITEMS_DIR, "/items"); + +(: canonical XML mirror collection: + - if $config:zotero-items-xml-dir starts with /db/ use as-is + - else treat it as relative to GROUP_BASE (strip leading / if present) :) +declare variable $zotero:XML_DIR as xs:string := + let $cfg := normalize-space($config:zotero-items-xml-dir) + return + if (starts-with($cfg, "/db/")) then $cfg + else concat($zotero:GROUP_BASE, "/", replace($cfg, "^/", "")); + +(: meta.json absolute resource path as configured :) +declare variable $zotero:META_PATH as xs:string := $config:zotero-meta-path; + +(: API constants :) +declare variable $zotero:API_BASE as xs:string := $config:zotero-api-base; +declare variable $zotero:GROUP_ID as xs:string := string($config:zotero-group-id); +declare variable $zotero:STYLE as xs:string := $config:zotero-style; +declare variable $zotero:API_KEY as xs:string := ($config:zotero-api-key, "")[1]; + +(: Call once (e.g. from post-install) to assert config and create needed collections. :) +declare function zotero:assert-config() as map(*) { + let $problems := + ( + if (not(starts-with($zotero:ITEMS_DIR, "/db/"))) then "items-dir must be an absolute collection" else (), + if (not(starts-with($zotero:XML_DIR, "/db/"))) then "xml-dir must be an absolute collection" else (), + if (not(xmldb:collection-available($zotero:GROUP_BASE))) + then concat("group-base missing: ", $zotero:GROUP_BASE) else () + ) + return + if (exists($problems)) then + map{ "ok": false(), "errors": array{ $problems } } + else ( + (: ensure items/xml collections exist; do it once here, not in hot paths :) + if (not(xmldb:collection-available($zotero:ITEMS_DIR))) then + xmldb:create-collection($zotero:ITEMS_DIR, "") else (), + if (not(xmldb:collection-available($zotero:XML_DIR))) then + xmldb:create-collection($zotero:XML_DIR, "") else (), + map{ + "ok": true(), + "itemsDir": $zotero:ITEMS_DIR, + "xmlDir": $zotero:XML_DIR, + "meta": $zotero:META_PATH + } + ) }; - -(: grab header values case-insensitively :) -declare %private function zotero:header($resp as element(http:response), $name as xs:string) as xs:string* { - $resp/http:header[lower-case(@name) = lower-case($name)]/@value/string() +declare %private function zotero:_xml-matches($i as element(item), $q as xs:string) as xs:boolean { + if ($q = "") then true() + else + let $lc := lower-case#1 + let $hay := string-join(( + $lc(string($i/title)), + for $c in $i/creators/c return string-join(($lc(string($c/@last)), $lc(string($c/@first)), $lc(string($c/@name))), " "), + $lc(string($i/doi)) + ), " ") + return contains($hay, $q) }; -(:~ - Build standard HTTP headers for Zotero API calls. - - Always includes: - - Accept: application/json - - User-Agent: eXist/zotero-sync - - Optionally includes: - - Authorization / Zotero-API-Key header if configured in your config.xqm. - - Any extra header passed in (e.g., If-Modified-Since-Version). - - Parameters: - @param $extra element(http:header)? Optional extra header to include. - - Return: - @return element(http:header)* A sequence of http:header elements ready to - be inserted into . - - Example: - - { attribute href { $href } } - { zotero:headers( - if ($since gt 0) - then - else () - ) - } - -:) -(: Build headers for Zotero requests :) -declare function zotero:headers($extra as element(http:header)?) as element(http:header)* { +(: ====== HEADERS for Zotero HTTP requests ====== :) +declare %private function zotero:headers($extra as element(http:header)?) as element(http:header)* { let $base := (, , - (: avoid gzip auto-encoding issues :) ) - let $key := normalize-space(string(($config:zotero-api-key, "")[1])) - let $auth := - if ($key ne "") - then - else () + let $auth := if (normalize-space($zotero:API_KEY) ne "") + then + else () return ($base, $auth, $extra) }; - -(: ───────────────── path helper (normalized) ───────────────── :) -declare %private function zotero:path-split($abs as xs:string) as map(*) { - let $norm := replace($abs, '/+$', '') (: drop trailing '/' :) - let $name := tokenize($norm, '/')[last()] - let $coll := substring($norm, 1, string-length($norm) - string-length($name) - 1) - return map{ "coll": $coll, "name": $name } -}; - -declare %private function zotero:resource-exists($coll as xs:string, $name as xs:string) as xs:boolean { - if (not(xmldb:collection-available($coll))) then false() - else some $r in xmldb:get-child-resources($coll) satisfies ($r = $name) -}; - -(:~ - Read the sync meta information from `$config:zotero-meta-path`. - - If the meta file does not exist or cannot be parsed, returns a default map: - { "libraryVersion": 0 } - - Return: - @return map(*) e.g. { "libraryVersion": 8306, "syncedAt": "2025-10-09T12:34:56Z" } - - Errors: - - Exceptions are caught internally; a default map is returned. -:) declare %private function zotero:read-meta() as map(*) { - let $ps := zotero:path-split($config:zotero-meta-path) - let $exists := zotero:resource-exists($ps?coll, $ps?name) - return - if (not($exists)) then - map{ "libraryVersion": 0, "syncedAt": "" } - else - let $uri := concat($ps?coll, "/", $ps?name) - let $bin := try { util:binary-doc($uri) } catch * { () } - let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" - return - if (normalize-space($txt) = "") then - map{ "libraryVersion": 0, "syncedAt": "" } - else - try { parse-json($txt) } catch * { map{ "libraryVersion": 0, "syncedAt": "" } } + if (doc-available($zotero:META_PATH)) then + let $bin := util:binary-doc($zotero:META_PATH) + let $txt := if ($bin) then util:binary-to-string($bin) else "" + return if ($txt ne "") then try { parse-json($txt) } catch * { map{ "libraryVersion": 0 } } + else map{ "libraryVersion": 0 } + else map{ "libraryVersion": 0 } }; -(:~ - Write sync meta information to `$config:zotero-meta-path`. - - Overwrites or creates `meta.json` with: - { - "libraryVersion": , - "syncedAt": current-dateTime() - } - - Parameters: - @param $libraryVersion xs:integer Zotero Last-Modified-Version to persist. - - Return: - @return xs:boolean true() on success, false() on any error. - - Side-effects: - - Stores JSON with media type `application/json`. -:) -declare %private function zotero:write-meta($lmv as xs:integer) as xs:boolean { - let $ps := zotero:path-split($config:zotero-meta-path) - let $json := serialize( - map{ "libraryVersion": $lmv, "syncedAt": current-dateTime() }, - map{ "method":"json", "indent": true() } - ) - let $_rm := try { xmldb:remove($ps?coll, $ps?name) } catch * { () } +declare %private function zotero:write-meta($lv as xs:integer) as xs:boolean { + let $coll := substring-before($zotero:META_PATH, concat("/", tokenize($zotero:META_PATH, "/")[last()])) + let $name := tokenize($zotero:META_PATH, "/")[last()] return try { - let $_ := xmldb:store($ps?coll, $ps?name, $json, "application/json") + let $_ := xmldb:store( + $coll, + $name, + serialize( + map{ "libraryVersion": $lv, "syncedAt": current-dateTime() }, + map{ "method":"json", "indent": true() } + ), + "application/json" + ) return true() } catch * { false() } }; -(: store one item .json into items dir — 4-arg store :) -declare %private function zotero:store-item($key as xs:string, $data as map(*)) as xs:string { - xmldb:store( - $config:zotero-items-dir, - concat($key, ".json"), - serialize($data, map{"method":"json","indent": true()}), - "application/json" - ) +(: ====== XML mirror helpers (one file per item) ====== :) +declare %private function zotero:xml-ensure-collection() as empty-sequence() { + if (not(xmldb:collection-available($config:zotero-items-xml-dir))) then + xmldb:create-collection($config:zotero-items-xml-dir, "") + else () }; -(:~ - Ingest a single Zotero page (array of items) into the local cache. - - For each array entry, extracts: - - key = item key (from top-level `key` or `data?key`) - - data = the `data` object - - bib = the `bib` string (when `include=bib` was requested) - - Stores `{ data + "bib": }` as `.json` under `$config:zotero-items-dir`. - - Parameters: - @param $arr array(*) The parsed JSON array from Zotero. - - Return: - @return xs:integer Number of items successfully stored. -:) -declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { - let $n := array:size($arr) +declare %private function zotero:xml-from-json($data as map(*), $bib as xs:string?) as element(item) { + let $key := string(($data?key, $data?data?key)[1]) + let $title := string(($data?title, $data?data?title)[1]) + let $dt := string(($data?dateModified, $data?data?dateModified)[1]) + let $parent := string(($data?parentItem, $data?data?parentItem)[1]) + let $type := string(($data?itemType, $data?data?itemType)[1]) + let $cre := $data?data?creators + let $tagsArr := $data?data?tags + let $doi := string(($data?data?DOI, $data?data?doi)[1]) return - if ($n = 0) then 0 - else - sum( - for $i in 1 to $n - let $item := array:get($arr, $i) - let $key := string(($item?key, $item?data?key)[1]) - let $data := $item?data - let $bib := $item?bib - let $toSave := - if (exists($data)) then - if (exists($bib)) then map:merge(($data, map{"bib": string($bib)})) - else $data - else - (: extremely rare, but if Zotero returned only a bib :) - map{"bib": string($bib)} - let $_ := - if ($key != "") then zotero:store-item($key, $toSave) else () - return if ($key != "") then 1 else 0 - ) -}; -declare %private function zotero:next-link($resp as element(http:response)) as xs:string { - let $links := $resp/http:header[lower-case(@name) = 'link']/@value/string() - let $cands := - for $line in $links - let $parts := tokenize($line, ',') - for $p in $parts - where contains($p, 'rel="next"') or contains($p, "rel='next'") - let $u := normalize-space(substring-before(substring-after($p, '<'), '>')) - return $u - return string-join((($cands)[1]), '') (: () -> '' :) + + { $title } + { + if ($cre instance of array(*)) then + for $i in 1 to array:size($cre) + let $c := array:get($cre, $i) + return element c { + if ($c?lastName) then attribute last { string($c?lastName) } else (), + if ($c?firstName) then attribute first { string($c?firstName) } else (), + if ($c?name) then attribute name { string($c?name) } else () + } + else () + } + { if (normalize-space($doi) ne "") then { $doi } else () } + { + if ($tagsArr instance of array(*)) then + for $i in 1 to array:size($tagsArr) + let $t := lower-case(normalize-space(string(array:get($tagsArr, $i)?tag))) + where $t ne "" return { $t } + else () + } + { if (normalize-space($bib) ne "") then { $bib } else } + }; -(:~ - Follow pagination and ingest subsequent pages. - - Issues an HTTP GET to `$next`, parses JSON, ingests items, and recursively - follows the next `Link: rel="next"` until exhausted. - - Parameters: - @param $next xs:string Absolute Zotero API URL taken from the Link header. - @param $acc xs:integer Accumulator of items ingested so far. +declare %private function zotero:xml-upsert( + $data as map(*), + $bib as xs:string? +) as empty-sequence() { + let $key := string(($data?key, $data?data?key)[1]) + let $xml := zotero:xml-from-json($data, $bib) + let $name := concat($key, ".xml") + let $_ := xmldb:store($zotero:XML_DIR, $name, $xml, "application/xml") + return () +}; - Return: - @return xs:integer Total count of items ingested (including prior pages). +(: ====== Ingest one page of items from Zotero (array) ====== :) +declare %private function zotero:ingest-page($arr as array(*)) as xs:integer { + let $n := + sum( + for $i in 1 to array:size($arr) + let $entry := array:get($arr, $i) + let $key := string(($entry?key, $entry?data?key)[1]) + let $data := if ($entry?data instance of map(*)) then $entry?data else map{} + let $bib := string(($entry?bib)[1]) + let $json := serialize(map{ "data": $data, "bib": $bib }, map{"method":"json"}) + let $_js := xmldb:store($config:zotero-items-dir, concat($key, ".json"), $json, "application/json") + let $_xml := zotero:xml-upsert(map{ "key": $key, "data": $data }, $bib) + return 1 + ) + return $n +}; - Notes: - - Pass an empty string to stop: the function will return $acc unchanged. - - Uses the same headers as the first page (zotero:headers()). -:) -declare %private function zotero:sync-follow($next as xs:string?, $acc as xs:integer) as xs:integer { - if (empty($next) or $next = '') then $acc +(: ====== Follow pagination via Link rel="next" and ingest ====== :) +declare %private function zotero:sync-follow($next as xs:string, $acc as xs:integer) as xs:integer { + if (normalize-space($next) = "") then $acc else let $req := - - { zotero:headers(()) } - - let $seq := try { http:send-request($req) } catch * { () } + + { attribute href { $next } } + { zotero:headers(()) } + + let $res := try { http:send-request($req) } catch * { () } return - if (empty($seq)) then $acc + if (empty($res)) then $acc else - let $resp := $seq[1] - let $status := xs:integer($resp/@status) + let $r1 := $res[1] + let $code := xs:integer($r1/@status) return - if ($status != 200) then $acc + if ($code != 200) then $acc else - let $raw := try { util:binary-to-string($seq[2]) } catch * { "" } - let $arr := if (normalize-space($raw) = "") then array{} else - try { parse-json($raw) } catch * { array{} } - let $c := zotero:ingest-page($arr) - let $more := zotero:next-link($resp) - return zotero:sync-follow($more, $acc + $c) + let $raw := try { util:binary-to-string($res[2]) } catch * { "" } + let $arr := if (normalize-space($raw) = "") then array{} else try { parse-json($raw) } catch * { array{} } + let $cnt := if ($arr instance of array(*)) then zotero:ingest-page($arr) else 0 + let $nextHref := + string(( + for $line in $r1/http:header[lower-case(@name)='link']/@value/string() + let $parts := tokenize($line, ",") + for $p in $parts + where contains($p, 'rel="next"') or contains($p, "rel='next'") + return normalize-space(substring-before(substring-after($p, "<"), ">")) + )[1]) + return zotero:sync-follow($nextHref, $acc + $cnt) }; -(: ───────────────────────────────────────────────────────── - Public endpoint: POST /api/zotero/sync - ───────────────────────────────────────────────────────── :) -declare function zotero:sync() as xs:string { - response:set-header("Content-Type","application/json"), - zotero:sync(map{}, ) -}; -declare function zotero:sync($request as map(*)) as xs:string { - response:set-header("Content-Type","application/json"), - zotero:sync($request, ) -}; - -(:~ - Sync cached Zotero items for the configured group. - - Hits the Zotero Web API `/groups/{groupId}/items` with `include=data,bib` - (plus your CSL `style`) and `limit=100`, follows pagination via the HTTP - `Link: rel="next"` header, and writes each item as `.json` into - `$config:zotero-items-dir`. Also updates `$config:zotero-meta-path` - with the latest `libraryVersion` and `syncedAt`. - - NOTE: Returns a serialized JSON string (xs:string) intentionally to avoid - Roaster’s atomization issue with map/array results. `Content-Type` is set - to `application/json`. - - Parameters: - @param $request map(*) Roaster request context (not used here; kept for arity). - @param $root element() Roaster root element (unused). +(: ====== SYNC endpoint ====== :) +declare function zotero:sync() as xs:string { zotero:sync(map{}, ) }; +declare function zotero:sync($request as map(*)) as xs:string { zotero:sync($request, ) }; - Return: - @return xs:string JSON string like: - { - "status":"ok|error", - "updated": , (: number of items written this call :) - "libraryVersion": , (: Zotero Last-Modified-Version :) - "metaWriteOk": true|false, (: meta.json write result :) - "metaPath": "/meta.json", - "httpStatus": , (: only on error :) - "errorBody": "", (: only on error :) - "requestHref": "" (: on error, sometimes on ok :) - } - - Headers: - - Sets `Content-Type: application/json`. - - Sends `If-Modified-Since-Version` when a previous `libraryVersion` exists. - - Adds `Zotero-API-Key` if configured in your module. - - Side-effects: - - Writes/overwrites `/.json` (merged { data + bib }). - - Writes/overwrites `meta.json` with { libraryVersion, syncedAt }. - - Status codes: - - 200 (body "status":"ok") on success or 304-from-Zotero. - - 200 (body "status":"error") with details on upstream HTTP errors. - - Example (curl): - curl -sS '…/api/zotero/sync' - - See also: - - zotero:headers() - - zotero:ingest-page() - - zotero:write-meta() - - zotero:sync-follow() -:) -declare function zotero:sync($request as map(*), $root as element()) { +declare function zotero:sync($request as map(*), $root as element()) as xs:string { response:set-header("Content-Type","application/json"), - let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } - let $since := xs:integer(($meta?libraryVersion, 0)[1]) - - let $base := concat($config:zotero-api-base, "/groups/", string($config:zotero-group-id), "/items") - - (: IMPORTANT: & must be & inside attributes; style value must be encoded :) - let $href := concat( - $base, - "?since=", encode-for-uri(string($since)), - "&limit=100", - "&include=data,bib", - "&format=json", - "&style=",$config:zotero-style - ) + let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } + let $since := xs:integer(($meta?libraryVersion, 0)[1]) + + let $base := concat($zotero:API_BASE, "/groups/", $zotero:GROUP_ID, "/items") + let $qs := string-join(( + concat("since=", encode-for-uri(string($since))), + "limit=100", + "include=data,bib", + "format=json", + if (normalize-space($zotero:STYLE) ne "") + then concat("style=", encode-for-uri($zotero:STYLE)) + else () + ), "&") + let $href := concat($base, "?", $qs) let $req := - + + { attribute href { $href } } { zotero:headers( if ($since gt 0) @@ -437,423 +264,180 @@ declare function zotero:sync($request as map(*), $root as element()) { let $respSeq := try { http:send-request($req) } catch * { () } - let $payload := + return if (empty($respSeq)) then - map{ - "status" : "error", - "reason" : "http:send-request failed", - "requestHref" : $href, - "metaPath" : $config:zotero-meta-path - } + serialize(map{ + "status":"error", + "reason":"http:send-request failed", + "requestHref": $href + }, map{"method":"json","indent":true()}) else let $resp := $respSeq[1] let $status := xs:integer($resp/@status) return - if ($status = 304) then - let $ok := zotero:write-meta($since) - return map{ - "status" : "ok", - "updated" : 0, - "libraryVersion" : $since, - "metaWriteOk" : $ok, - "metaPath" : $config:zotero-meta-path - } + + if ($status = 304) then ( + zotero:write-meta($since), + serialize(map{ "status":"ok", "updated": 0, "libraryVersion": $since }, + map{"method":"json","indent":true()}) + ) else if ($status != 200) then - let $errBody := try { util:binary-to-string($respSeq[2]) } catch * { "" } - return map{ - "status" : "error", - "httpStatus" : $status, - "errorBody" : $errBody, - "requestHref" : $href, - "metaPath" : $config:zotero-meta-path - } + let $err := try { util:binary-to-string($respSeq[2]) } catch * { "" } + return serialize(map{ + "status":"error", + "httpStatus": $status, + "errorBody": $err, + "requestHref": $href + }, map{"method":"json","indent":true()}) else let $raw := try { util:binary-to-string($respSeq[2]) } catch * { "" } let $clean := if (starts-with($raw, codepoints-to-string(65279))) then substring($raw, 2) else $raw - let $arr := if (normalize-space($clean) = "") then array{} else - try { parse-json($clean) } catch * { array{} } - let $items := if ($arr instance of array(*)) then $arr else array{} - let $c1 := zotero:ingest-page($items) - - (: make $next a STRING so we never pass () into sync-follow :) + let $arr := if (normalize-space($clean) = "") then array{} else try { parse-json($clean) } catch * { array{} } + let $c1 := + if ($arr instance of array(*)) then + sum( + for $i in 1 to array:size($arr) + let $entry := array:get($arr, $i) + let $key := string(($entry?key, $entry?data?key)[1]) + let $data := if ($entry?data instance of map(*)) then $entry?data else map{} + let $bib := string(($entry?bib)[1]) + let $json := serialize(map{ "data": $data, "bib": $bib }, map{"method":"json"}) + let $_j := xmldb:store($zotero:ITEMS_DIR, concat($key, ".json"), $json, "application/json") + let $_x := zotero:xml-upsert(map{ "key": $key, "data": $data }, $bib) + return 1 + ) + else 0 let $next := string(( - for $line in $resp/http:header[lower-case(@name)='link']/@value/string() - let $parts := tokenize($line, ",") - for $p in $parts - where contains($p, 'rel="next"') or contains($p, "rel='next'") - return normalize-space(substring-before(substring-after($p, "<"), ">")) - )[1]) - - let $cN := if ($next = '') then 0 else zotero:sync-follow($next, 0) - + for $line in $resp/http:header[lower-case(@name)='link']/@value/string() + let $parts := tokenize($line, ",") + for $p in $parts + where contains($p, 'rel="next"') or contains($p, "rel='next'") + return normalize-space(substring-before(substring-after($p, "<"), ">")) + )[1]) + let $cN := if ($next = '') then 0 else zotero:sync-follow($next, 0) let $lmvStr := ($resp/http:header[lower-case(@name)='last-modified-version']/@value)[1] - let $lmv := if (exists($lmvStr) and normalize-space($lmvStr) ne "") then xs:integer($lmvStr) else $since - - let $ok := zotero:write-meta($lmv) - - return map{ - "status" : "ok", - "updated" : $c1 + $cN, - "libraryVersion" : $lmv, - "metaWriteOk" : $ok, - "metaPath" : $config:zotero-meta-path - } - - return serialize($payload, map{ "method":"json", "indent": true() }) + let $lmv := if ($lmvStr and normalize-space($lmvStr) ne "") then xs:integer($lmvStr) else $since + let $_m := zotero:write-meta($lmv) + return serialize(map{ + "status":"ok", + "updated": $c1 + $cN, + "libraryVersion": $lmv + }, map{"method":"json","indent":true()}) }; -(: ───────────────────────────────────────────────────────── - Public endpoint: POST /api/zotero/items/search - ───────────────────────────────────────────────────────── :) +(: ====== SUGGEST (lightweight for autocomplete) ====== :) +declare function zotero:items-suggest() as xs:string { zotero:items-suggest(map{}, ) }; +declare function zotero:items-suggest($request as map(*)) as xs:string { zotero:items-suggest($request, ) }; -declare function zotero:items-search() as xs:string { - zotero:items-search(map{}, ) -}; - -declare function zotero:items-search($request as map(*)) as xs:string { - zotero:items-search($request, ) +declare function zotero:items-suggest($request as map(*), $root as element()) as xs:string { + response:set-header("Content-Type", "application/json"), + let $q := lower-case(normalize-space(request:get-parameter("q", ""))) + let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) + let $limit := let $l := number(request:get-parameter("limit", "8")) return if ($l ge 1) then xs:integer($l) else 8 + + let $pool := + collection($zotero:XML_DIR)/item[ + zotero:_xml-matches(., $q) + and ( $tag = "" or tags/tag = $tag ) + ] + let $sorted := for $i in $pool order by xs:dateTime($i/@dateModified) descending return $i + let $picked := subsequence($sorted, 1, $limit) + + let $arr := array { + for $i in $picked + return map{ + "key": string($i/@key), + "title": string($i/title), + "bib": if ($i/bib/@html = "true") then string($i/bib) else "" + } + } + return serialize($arr, map{ "method":"json", "indent": true() }) }; -(:~ - Search cached items (JSON only, from local cache). - - Scans `$config:zotero-items-dir` for `*.json`, applies optional - full-text query `q` over title/creators/DOI, optional `tag` filter, - applies `limit`, and returns a result object with counts and items. - - NOTE: Returns a serialized JSON string (xs:string). Sets - `Content-Type: application/json` and `X-Total-Count`. - - Query parameters: - - q (string) Full-text across title + creators + DOI (case-insensitive). - - tag (string) Match items that have this Zotero tag (case-insensitive). - - limit (integer) Maximum items to return (default 15; minimum 1). - - Parameters: - @param $request map(*) Roaster request context (unused beyond reading query params). - @param $root element() Roaster root element (unused). - - Return: - @return xs:string JSON like: - { - "query": { "q":"…", "tag":"…", "limit": 15 }, - "total": , (: matches before limit :) - "returned": , (: items included :) - "items": [ { "key":"…", "data": {…} }, … ] - } +declare function zotero:items-search() as xs:string { zotero:items-search(map{}, ) }; +declare function zotero:items-search($request as map(*)) as xs:string { zotero:items-search($request, ) }; - Matching rules: - - `q` is matched with `contains()` against a normalized string built from: - - title (data?title) - - creators (joined first/last/name) - - DOI (data?DOI or data?doi) - - `tag` matches if the item has ANY tag equal to the parameter (lowercased). - - Headers: - - `Content-Type: application/json` - - `X-Total-Count: ` - - Status codes: - - 200 (always) — empty result set when nothing matches. - -:) declare function zotero:items-search($request as map(*), $root as element()) as xs:string { response:set-header("Content-Type", "application/json"), - - let $coll := $config:zotero-items-dir - let $qIn := lower-case(normalize-space(request:get-parameter("q", ""))) - let $tagIn := lower-case(normalize-space(request:get-parameter("tag", ""))) - let $limIn := $request?parameters?limit - let $limit := let $n := try { xs:integer($limIn) } catch * { 15 } - return if ($n lt 1) then 15 else $n - - let $names := - if (xmldb:collection-available($coll)) - then for $n in xmldb:get-child-resources($coll) - where ends-with($n, ".json") - return $n - else () - - let $matches := - for $name in $names - let $uri := concat($coll, "/", $name) - let $bin := try { util:binary-doc($uri) } catch * { () } - let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" - where string-length($txt) gt 0 - let $data := try { parse-json($txt) } catch * { map{} } - - let $key := string(( $data?key, replace($name, "\.json$", "") )[1]) - - (: tag filter :) - let $hasTag := - if ($tagIn = "") then true() - else if ($data?tags instance of array(*)) then - some $i in 1 to array:size($data?tags) - satisfies lower-case(string((array:get($data?tags, $i)?tag)[1])) = $tagIn - else false() - - (: q filter over title + creators + DOI — SAFE coalescing :) - let $title := lower-case(string(($data?title)[1])) - let $creators := - if ($data?creators instance of array(*)) then - string-join( - for $i in 1 to array:size($data?creators) - let $c := array:get($data?creators, $i) - let $ln := string(($c?lastName)[1]) - let $fn := string(($c?firstName)[1]) - let $nm := string(($c?name)[1]) - let $parts := ($ln, $fn, $nm) - let $nonEmpty := for $p in $parts where normalize-space($p) ne "" return $p - let $one := normalize-space(string-join($nonEmpty, " ")) - where $one ne "" - return $one - , " ") - else "" - let $doi := lower-case(string((($data?DOI, $data?doi)[1]))) - - let $hay := normalize-space(string-join(($title, $creators, $doi), " ")) - let $okQ := ($qIn = "") or contains($hay, $qIn) - - where $hasTag and $okQ - return map{ "key": $key, "data": $data } - - let $total := count($matches) - let $limited := subsequence($matches, 1, $limit) - let $_hdr := response:set-header("X-Total-Count", string($total)) - - let $payload := map{ - "query": map{ "q": $qIn, "tag": $tagIn, "limit": $limit }, - "total": $total, (: matches before limit :) - "returned": count($limited), (: items in this page :) - "items": array { $limited } (: [{key,data}, …] :) + let $q := lower-case(normalize-space(request:get-parameter("q", ""))) + let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) + let $limit := let $l := number(request:get-parameter("limit", "15")) return if ($l ge 1) then xs:integer($l) else 15 + + let $pool := + collection($zotero:XML_DIR)/item[ + ( $q = "" or ft:query(., $q) ) + and ( $tag = "" or tags/tag = $tag ) + ] + + let $sorted := for $i in $pool order by xs:dateTime($i/@dateModified) descending return $i + let $total := count($sorted) + let $picked := subsequence($sorted, 1, $limit) + + let $items := array { + for $i in $picked + let $key := string($i/@key) + let $bin := util:binary-doc(concat($zotero:ITEMS_DIR, "/", $key, ".json")) + let $txt := if ($bin) then util:binary-to-string($bin) else "" + let $obj := if ($txt ne "") then try { parse-json($txt) } catch * { () } else () + return + if ($obj instance of map(*)) then map{ "key": $key, "data": ($obj?data, $obj)[1] } + else map{ "key": $key, "data": map{} } } - return serialize($payload, map{ "method": "json", "indent": true() }) -}; - -(: ───────────────────────────────────────────────────────── - Public endpoint: POST /api/zotero/items/bib - ───────────────────────────────────────────────────────── :) - -(: ── wrappers (streaming: return empty-sequence()) ── :) -declare function zotero:item-bib() as empty-sequence() { - zotero:item-bib(map{}, ) + let $resp := map{ + "query": map{ "q": $q, "tag": $tag, "limit": $limit }, + "total": $total, + "returned": array:size($items), + "items": $items + } + return serialize($resp, map{ "method":"json", "indent": true() }) }; -declare function zotero:item-bib($request as map(*)) as empty-sequence() { - zotero:item-bib($request, ) -}; +(: ====== BIB HTML SNIPPET (streams raw HTML) ====== :) +declare function zotero:item-bib() as empty-sequence() { zotero:item-bib(map{}, ) }; +declare function zotero:item-bib($request as map(*)) as empty-sequence() { zotero:item-bib($request, ) }; -(: helper: serialize a small HTML fallback safely :) declare %private function zotero:_fallback-html($title as xs:string) as xs:string { - serialize( - { $title }, - map{ "method":"html", "omit-xml-declaration": true(), "indent": false() } - ) + serialize({ $title }, map{"method":"html","omit-xml-declaration":true(),"indent":false()}) }; -(: helper: load one cached item by key (returns parsed map or empty) :) -declare %private function zotero:_load-item-by-key($key as xs:string) as item()? { - let $coll := $config:zotero-items-dir - let $uri := concat($coll, "/", $key, ".json") - let $bin := try { util:binary-doc($uri) } catch * { () } - let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" +declare %private function zotero:_load-json($key as xs:string) as map(*)? { + let $bin := util:binary-doc(concat($config:zotero-items-dir, "/", $key, ".json")) + let $txt := if ($bin) then util:binary-to-string($bin) else "" return if ($txt ne "") then try { parse-json($txt) } catch * { () } else () }; -(: ── MAIN: GET /api/zotero/items/top/bib?key=... | ?tag=... ── :) declare function zotero:item-bib($request as map(*), $root as element()) as empty-sequence() { response:set-header("Content-Type", "text/html; charset=UTF-8"), - let $coll := $config:zotero-items-dir let $key := normalize-space(request:get-parameter("key", "")) let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) - - let $emit := - function($html as xs:string) as empty-sequence() { - response:stream-binary(util:string-to-binary($html), "text/html", ()), - () - } + let $emit := function($html as xs:string) as empty-sequence() { + response:stream-binary(util:string-to-binary($html), "text/html", ()), () + } return - if (not(xmldb:collection-available($coll))) then ( - response:set-status-code(404), - $emit("") - ) - - else if ($key ne "") then - let $item := zotero:_load-item-by-key($key) + if ($key ne "") then + let $x := doc(concat($config:zotero-items-xml-dir, "/", $key, ".xml"))/item return - if (empty($item)) then ( - response:set-status-code(404), - $emit("") - ) else - let $bib := string(($item?bib)[1]) + if (empty($x)) then ( response:set-status-code(404), $emit("") ) + else + let $bib := if ($x/bib/@html="true") then string($x/bib) else "" return - if (normalize-space($bib) ne "") then - $emit($bib) + if ($bib ne "") then $emit($bib) else - let $parentKey := string(($item?parentItem)[1]) - return - if (normalize-space($parentKey) ne "") then - let $parent := zotero:_load-item-by-key($parentKey) - return - if (empty($parent)) then - $emit(zotero:_fallback-html(string((($item?title, $item?note)[1])))) - else - let $pb := string(($parent?bib)[1]) - return - if (normalize-space($pb) ne "") then - $emit($pb) - else - $emit(zotero:_fallback-html(string((($parent?title, $item?title, $item?note)[1])))) - else - $emit(zotero:_fallback-html(string((($item?title, $item?note)[1])))) - + let $json := zotero:_load-json($key) + let $title := string(($json?data?title, "")[1]) + return $emit(zotero:_fallback-html($title)) else if ($tag ne "") then - let $names := xmldb:get-child-resources($coll)[ends-with(., ".json")] - let $found := - ( - for $name in $names - let $uri := concat($coll, "/", $name) - let $bin := try { util:binary-doc($uri) } catch * { () } - let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" - where $txt ne "" - let $data := try { parse-json($txt) } catch * { () } - - (: top-level only :) - let $parent := string(($data?parentItem)[1]) - where normalize-space($parent) = "" - - (: tag match, case-insensitive :) - let $tags := - if ($data?tags instance of array(*)) then - for $i in 1 to array:size($data?tags) - return lower-case(normalize-space(string((array:get($data?tags, $i)?tag)[1]))) - else () - where some $t in $tags satisfies $t = $tag - - return $data - )[1] + let $x := (collection($config:zotero-items-xml-dir)/item[normalize-space(@parentItem) = "" and tags/tag = $tag])[1] return - if (empty($found)) then ( - response:set-status-code(404), - $emit("") - ) + if (empty($x)) then ( response:set-status-code(404), $emit("") ) else - let $bib := string(($found?bib)[1]) + let $bib := if ($x/bib/@html="true") then string($x/bib) else "" return - if (normalize-space($bib) ne "") then - $emit($bib) - else - $emit(zotero:_fallback-html(string(($found?title)[1]))) - - else ( - response:set-status-code(400), - $emit("") - ) -}; - -(: ── wrappers ── :) -declare function zotero:items-suggest() as xs:string { - zotero:items-suggest(map{}, ) -}; - -declare function zotero:items-suggest($request as map(*)) as xs:string { - zotero:items-suggest($request, ) -}; - -(: serialize a tiny HTML safely :) -declare %private function zotero:_as-html($n as node()) as xs:string { - serialize($n, map { "method":"html", "omit-xml-declaration": true(), "indent": false() }) -}; - -(: MAIN: GET /api/zotero/items/suggest?q=&tag=&limit=&top=1 :) -declare function zotero:items-suggest($request as map(*), $root as element()) as xs:string { - response:set-header("Content-Type", "application/json"), - - let $coll := $config:zotero-items-dir - let $qIn := lower-case(normalize-space(request:get-parameter("q", ""))) - let $tagIn := lower-case(normalize-space(request:get-parameter("tag", ""))) - let $limIn := request:get-parameter("limit", "8") - let $limit := let $n := try { xs:integer($limIn) } catch * { 8 } - return if ($n lt 1) then 8 else $n - let $topIn := request:get-parameter("top", "1") - let $topOnly:= not($topIn = ("0","false","no")) - - let $names := - if (xmldb:collection-available($coll)) - then xmldb:get-child-resources($coll)[ends-with(., ".json")] - else () - - let $matches := - for $name in $names - let $uri := concat($coll, "/", $name) - let $bin := try { util:binary-doc($uri) } catch * { () } - let $txt := if (exists($bin)) then util:binary-to-string($bin) else "" - where $txt ne "" - let $data := try { parse-json($txt) } catch * { map{} } - - (: skip non-top-level if requested :) - let $parent := string(($data?parentItem)[1]) - where (not($topOnly)) or (normalize-space($parent) = "") - - (: quick tag set :) - let $tags := - if ($data?tags instance of array(*)) then - for $i in 1 to array:size($data?tags) - return lower-case(normalize-space(string((array:get($data?tags, $i)?tag)[1]))) - else () - let $primaryTag := string(($tags[1], "")[1]) - - (: tag filter :) - where ($tagIn = "") or (some $t in $tags satisfies $t = $tagIn) - - (: build haystack for q over title/creators/DOI :) - let $title := lower-case(string(($data?title)[1])) - let $creators := - if ($data?creators instance of array(*)) then - string-join( - for $i in 1 to array:size($data?creators) - let $c := array:get($data?creators, $i) - let $ln := string(($c?lastName)[1]) - let $fn := string(($c?firstName)[1]) - let $nm := string(($c?name)[1]) - let $parts := ($ln, $fn, $nm) - let $nonEmpty := for $p in $parts where normalize-space($p) ne "" return $p - let $one := normalize-space(string-join($nonEmpty, " ")) - where $one ne "" - return $one - , " ") - else "" - let $doi := lower-case(string((($data?DOI, $data?doi)[1]))) - let $hay := normalize-space(string-join(($title, $creators, $doi), " ")) - - where ($qIn = "") or contains($hay, $qIn) - - (: display label: cached bib or title fallback :) - let $bib := string(($data?bib)[1]) - let $label := - if (normalize-space($bib) ne "") then $bib - else zotero:_as-html({ $title }) - - let $key := string(($data?key, replace($name, "\.json$", ""))[1]) - - return map{ - "key": $key, - "tag": $primaryTag, - "label": $label - } - - let $total := count($matches) - let $limited := subsequence($matches, 1, $limit) - - let $payload := map{ - "query": map{ "q": $qIn, "tag": $tagIn, "limit": $limit, "top": $topOnly }, - "total": $total, - "returned": count($limited), - "items": array { $limited } - } - - return serialize($payload, map{ "method":"json", "indent": true() }) + if ($bib ne "") then $emit($bib) + else $emit(zotero:_fallback-html(string($x/title))) + else + ( response:set-status-code(400), $emit("") ) }; diff --git a/src/post-install.xql b/src/post-install.xql index 57713e0..434fcea 100644 --- a/src/post-install.xql +++ b/src/post-install.xql @@ -5,6 +5,7 @@ import module namespace pmc="http://www.tei-c.org/tei-simple/xquery/config"; import module namespace odd="http://www.tei-c.org/tei-simple/odd2odd"; import module namespace config="http://www.tei-c.org/tei-simple/config" at "modules/config.xqm"; import module namespace tpu="http://www.tei-c.org/tei-publisher/util" at "util.xql"; +import module namespace zotero="http://e-editiones.org/edep/api/zotero" at "modules/lib/api/zotero.xql"; declare namespace repo="http://exist-db.org/xquery/repo"; @@ -167,6 +168,7 @@ declare function local:zotero-ensure-layout() as map(*) { let $_b := local:mkcol-abs($config:zotero-base-dir) let $_g := local:mkcol-abs($config:zotero-group-dir) let $_i := local:mkcol-abs($config:zotero-items-dir) + let $_h := local:mkcol-abs($config:zotero-items-xml-dir) let $metaSeeded := local:zotero-seed-meta-if-missing($config:zotero-meta-path) @@ -176,12 +178,14 @@ declare function local:zotero-ensure-layout() as map(*) { "base": xmldb:collection-available($config:zotero-base-dir), "group": xmldb:collection-available($config:zotero-group-dir), "items": xmldb:collection-available($config:zotero-items-dir), + "items-xml": xmldb:collection-available($config:zotero-items-xml-dir), "metaSeeded": $metaSeeded }, "paths": map{ "base": $config:zotero-base-dir, "group": $config:zotero-group-dir, "items": $config:zotero-items-dir, + "items-xml": $config:zotero-items-xml-dir, "meta": $config:zotero-meta-path } } @@ -201,6 +205,7 @@ local:mkcol($target, "transform"), local:generate-code($target), local:create-data-collection(), local:zotero-ensure-layout(), +zotero:assert-config(), xmldb:reindex('/db/apps/edep-data'), let $pmuConfig := pmc:generate-pm-config(($config:odd-available, $config:odd-internal), $config:default-odd, $config:odd-root) return From c5388c485f4eedd44c1bf093e799e0ce8c93d95f Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Wed, 15 Oct 2025 18:04:46 +0200 Subject: [PATCH 091/254] cleanup --- src/modules/custom-api.json | 4 ++++ src/post-install.xql | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index cf775b9..c249a5e 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -852,6 +852,7 @@ }, "/api/zotero/sync": { "get": { + "security": [{ "cookieAuth": [] }], "summary": "Incrementally sync local cache.", "tags": ["zotero"], "operationId": "zotero:sync", @@ -870,6 +871,7 @@ }, "/api/zotero/items/bib": { "get": { + "security": [{ "basicAuth": [] }], "summary": "Return a single cached item's bibliography (HTML) by key or tag.", "tags": ["zotero"], "operationId": "zotero:item-bib", @@ -888,6 +890,7 @@ }, "/api/zotero/items/suggest": { "get": { + "security": [{ "basicAuth": [] }], "summary": "Lightweight suggestions for autocomplete (key, tag, label HTML).", "tags": ["zotero"], "operationId": "zotero:items-suggest", @@ -907,6 +910,7 @@ }, "/api/zotero/items/search": { "get": { + "security": [{ "basicAuth": [] }], "summary": "Search cached items. JSON only.", "tags": ["zotero"], "operationId": "zotero:items-search", diff --git a/src/post-install.xql b/src/post-install.xql index 434fcea..a92dff3 100644 --- a/src/post-install.xql +++ b/src/post-install.xql @@ -5,7 +5,6 @@ import module namespace pmc="http://www.tei-c.org/tei-simple/xquery/config"; import module namespace odd="http://www.tei-c.org/tei-simple/odd2odd"; import module namespace config="http://www.tei-c.org/tei-simple/config" at "modules/config.xqm"; import module namespace tpu="http://www.tei-c.org/tei-publisher/util" at "util.xql"; -import module namespace zotero="http://e-editiones.org/edep/api/zotero" at "modules/lib/api/zotero.xql"; declare namespace repo="http://exist-db.org/xquery/repo"; @@ -205,7 +204,6 @@ local:mkcol($target, "transform"), local:generate-code($target), local:create-data-collection(), local:zotero-ensure-layout(), -zotero:assert-config(), xmldb:reindex('/db/apps/edep-data'), let $pmuConfig := pmc:generate-pm-config(($config:odd-available, $config:odd-internal), $config:default-odd, $config:odd-root) return From 025cda688a4481b1593a39ece2843673fb544ed5 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Wed, 15 Oct 2025 18:23:12 +0200 Subject: [PATCH 092/254] sync both json + xml, cleaned up --- src/modules/lib/api/zotero.xql | 58 +++++----------------------------- 1 file changed, 8 insertions(+), 50 deletions(-) diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 888912f..80bc78e 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -43,36 +43,10 @@ declare variable $zotero:META_PATH as xs:string := $config:zotero-meta-path; (: API constants :) declare variable $zotero:API_BASE as xs:string := $config:zotero-api-base; -declare variable $zotero:GROUP_ID as xs:string := string($config:zotero-group-id); +declare variable $zotero:GROUP_ID as xs:string := $config:zotero-group-id; declare variable $zotero:STYLE as xs:string := $config:zotero-style; -declare variable $zotero:API_KEY as xs:string := ($config:zotero-api-key, "")[1]; - -(: Call once (e.g. from post-install) to assert config and create needed collections. :) -declare function zotero:assert-config() as map(*) { - let $problems := - ( - if (not(starts-with($zotero:ITEMS_DIR, "/db/"))) then "items-dir must be an absolute collection" else (), - if (not(starts-with($zotero:XML_DIR, "/db/"))) then "xml-dir must be an absolute collection" else (), - if (not(xmldb:collection-available($zotero:GROUP_BASE))) - then concat("group-base missing: ", $zotero:GROUP_BASE) else () - ) - return - if (exists($problems)) then - map{ "ok": false(), "errors": array{ $problems } } - else ( - (: ensure items/xml collections exist; do it once here, not in hot paths :) - if (not(xmldb:collection-available($zotero:ITEMS_DIR))) then - xmldb:create-collection($zotero:ITEMS_DIR, "") else (), - if (not(xmldb:collection-available($zotero:XML_DIR))) then - xmldb:create-collection($zotero:XML_DIR, "") else (), - map{ - "ok": true(), - "itemsDir": $zotero:ITEMS_DIR, - "xmlDir": $zotero:XML_DIR, - "meta": $zotero:META_PATH - } - ) -}; +declare variable $zotero:API_KEY as xs:string := $config:zotero-api-key; + declare %private function zotero:_xml-matches($i as element(item), $q as xs:string) as xs:boolean { if ($q = "") then true() else @@ -126,13 +100,6 @@ declare %private function zotero:write-meta($lv as xs:integer) as xs:boolean { } }; -(: ====== XML mirror helpers (one file per item) ====== :) -declare %private function zotero:xml-ensure-collection() as empty-sequence() { - if (not(xmldb:collection-available($config:zotero-items-xml-dir))) then - xmldb:create-collection($config:zotero-items-xml-dir, "") - else () -}; - declare %private function zotero:xml-from-json($data as map(*), $bib as xs:string?) as element(item) { let $key := string(($data?key, $data?data?key)[1]) let $title := string(($data?title, $data?data?title)[1]) @@ -229,14 +196,12 @@ declare %private function zotero:sync-follow($next as xs:string, $acc as xs:inte }; (: ====== SYNC endpoint ====== :) -declare function zotero:sync() as xs:string { zotero:sync(map{}, ) }; -declare function zotero:sync($request as map(*)) as xs:string { zotero:sync($request, ) }; - -declare function zotero:sync($request as map(*), $root as element()) as xs:string { +declare function zotero:sync($request as map(*)) { response:set-header("Content-Type","application/json"), let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } let $since := xs:integer(($meta?libraryVersion, 0)[1]) + let $log := util:log('info','USER ' || sm:id()//sm:real/sm:username/string()) let $base := concat($zotero:API_BASE, "/groups/", $zotero:GROUP_ID, "/items") let $qs := string-join(( @@ -326,10 +291,7 @@ declare function zotero:sync($request as map(*), $root as element()) as xs:strin }; (: ====== SUGGEST (lightweight for autocomplete) ====== :) -declare function zotero:items-suggest() as xs:string { zotero:items-suggest(map{}, ) }; -declare function zotero:items-suggest($request as map(*)) as xs:string { zotero:items-suggest($request, ) }; - -declare function zotero:items-suggest($request as map(*), $root as element()) as xs:string { +declare function zotero:items-suggest($request as map(*)) { response:set-header("Content-Type", "application/json"), let $q := lower-case(normalize-space(request:get-parameter("q", ""))) let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) @@ -354,10 +316,8 @@ declare function zotero:items-suggest($request as map(*), $root as element()) as return serialize($arr, map{ "method":"json", "indent": true() }) }; -declare function zotero:items-search() as xs:string { zotero:items-search(map{}, ) }; -declare function zotero:items-search($request as map(*)) as xs:string { zotero:items-search($request, ) }; -declare function zotero:items-search($request as map(*), $root as element()) as xs:string { +declare function zotero:items-search($request as map(*)) { response:set-header("Content-Type", "application/json"), let $q := lower-case(normalize-space(request:get-parameter("q", ""))) let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) @@ -394,8 +354,6 @@ declare function zotero:items-search($request as map(*), $root as element()) as }; (: ====== BIB HTML SNIPPET (streams raw HTML) ====== :) -declare function zotero:item-bib() as empty-sequence() { zotero:item-bib(map{}, ) }; -declare function zotero:item-bib($request as map(*)) as empty-sequence() { zotero:item-bib($request, ) }; declare %private function zotero:_fallback-html($title as xs:string) as xs:string { serialize({ $title }, map{"method":"html","omit-xml-declaration":true(),"indent":false()}) @@ -407,7 +365,7 @@ declare %private function zotero:_load-json($key as xs:string) as map(*)? { return if ($txt ne "") then try { parse-json($txt) } catch * { () } else () }; -declare function zotero:item-bib($request as map(*), $root as element()) as empty-sequence() { +declare function zotero:item-bib($request as map(*)) { response:set-header("Content-Type", "text/html; charset=UTF-8"), let $key := normalize-space(request:get-parameter("key", "")) From 6afcda867fc0c4a0c764d0fa5af0aab067be0807 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 13:29:43 +0200 Subject: [PATCH 093/254] works without bib endpoint --- src/resources/scripts/zotero-autocomplete.js | 48 +++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index 3dff499..c90301e 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -225,12 +225,48 @@ class ZoteroAutocomplete extends HTMLElement { } } - async _fetchBib(key) { - const url = new URL(this._bibEndpoint, window.location.href); - url.searchParams.set('key', key); - const res = await fetch(url.toString(), { credentials: 'include' }); - if (!res.ok) throw new Error('HTTP ' + res.status); - return await res.text(); + // NEW: local helper to read bib from already loaded results (no network) + // NEW: local helper to read bib from already loaded results (no network) + _getBib(id) { + const needle = (id || '').trim().toLowerCase(); + if (!needle) return ''; + + const items = Array.isArray(this._results) ? this._results : []; + const hit = items.find(it => { + const tag = (it && it.tag ? String(it.tag) : '').toLowerCase(); + const key = (it && it.key ? String(it.key) : '').toLowerCase(); + return tag === needle || key === needle; + }); + return hit && typeof hit.bib === 'string' ? hit.bib : ''; + } + + // REPLACE your current _fetchBib with this version + async _fetchBib(id) { + // 1) try already-loaded suggestions + const local = this._getBib(id); + if (local) return local; + + // 2) fallback for preset values: resolve one item by tag + const tag = (id || '').trim(); + if (!tag) return ''; + + try { + // keep your existing endpoint; resolve relative to the current document URL + const url = new URL(this.api || '/exist/apps/edep/api/zotero/items/suggest', document.baseURI); + url.searchParams.set('tag', tag); + url.searchParams.set('limit', '1'); + + const resp = await fetch(url.toString(), { + headers: { Accept: 'application/json' }, + credentials: 'include', + }); + if (!resp.ok) return ''; + + const arr = await resp.json(); + return Array.isArray(arr) && arr.length && typeof arr[0].bib === 'string' ? arr[0].bib : ''; + } catch (_) { + return ''; + } } /* ===== render list ===== */ From 66b459440282bcb7133896731335182659ec5c1f Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 13:44:36 +0200 Subject: [PATCH 094/254] works without bib endpoint --- src/resources/scripts/zotero-autocomplete.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index c90301e..bd41bcc 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -252,7 +252,7 @@ class ZoteroAutocomplete extends HTMLElement { try { // keep your existing endpoint; resolve relative to the current document URL - const url = new URL(this.api || '/exist/apps/edep/api/zotero/items/suggest', document.baseURI); + const url = new URL(this._endpoint, document.baseURI); url.searchParams.set('tag', tag); url.searchParams.set('limit', '1'); From 0849606f52a8048b97777cc144c307ab6022f74b Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 13:45:52 +0200 Subject: [PATCH 095/254] works without bib endpoint --- src/templates/zotero-autocomplete.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/templates/zotero-autocomplete.html b/src/templates/zotero-autocomplete.html index 7280939..e98a39d 100644 --- a/src/templates/zotero-autocomplete.html +++ b/src/templates/zotero-autocomplete.html @@ -26,7 +26,7 @@ - JECUYVKL + schillinger-häfelevierternachtrag1977 @@ -36,7 +36,6 @@ From 6d198c409f73d31e343b704a2d0d7629ea9fac96 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 14:29:17 +0200 Subject: [PATCH 096/254] works without bib endpoint --- src/resources/scripts/zotero-autocomplete.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index bd41bcc..1fbc46d 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -1,5 +1,6 @@ -// resources/scripts/zotero-autocomplete.js -// Light-DOM web component with Fore-friendly events & multiline overlay +/* + Light-DOM web component with Fore-friendly events & multiline overlay + */ class ZoteroAutocomplete extends HTMLElement { static get observedAttributes() { return ['endpoint', 'bib-endpoint', 'tag', 'limit', 'minlength', 'debounce', 'value', 'name']; @@ -50,7 +51,6 @@ class ZoteroAutocomplete extends HTMLElement { connectedCallback() { // config this._endpoint = this.getAttribute('endpoint') || '/api/zotero/items/suggest'; - this._bibEndpoint = this.getAttribute('bib-endpoint') || '/api/zotero/items/bib'; this._tag = this.getAttribute('tag') || ''; this._limit = parseInt(this.getAttribute('limit') || '8', 10); this._minlen = parseInt(this.getAttribute('minlength') || String(this._minlen), 10); @@ -90,7 +90,6 @@ class ZoteroAutocomplete extends HTMLElement { attributeChangedCallback(name, _old, value) { if (!this.isConnected) return; if (name === 'endpoint') this._endpoint = value || this._endpoint; - if (name === 'bib-endpoint') this._bibEndpoint = value || this._bibEndpoint; if (name === 'tag') this._tag = value || ''; if (name === 'limit') this._limit = parseInt(value || '8', 10); if (name === 'minlength') this._minlen = parseInt(value || '2', 10); From 37d4d9884d260edc01e6156749e12dbce47397fb Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 16:19:57 +0200 Subject: [PATCH 097/254] fixed component again to use tag instead of key --- src/modules/lib/api/zotero.xql | 6 +- src/resources/scripts/zotero-autocomplete.js | 114 +++++++++++-------- 2 files changed, 74 insertions(+), 46 deletions(-) diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 80bc78e..7c21ad4 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -202,6 +202,8 @@ declare function zotero:sync($request as map(*)) { let $meta := try { zotero:read-meta() } catch * { map{ "libraryVersion": 0 } } let $since := xs:integer(($meta?libraryVersion, 0)[1]) let $log := util:log('info','USER ' || sm:id()//sm:real/sm:username/string()) + let $user := $request?user + let $log := util:log('info','REQUEST USER ' || sm:id()//sm:real/sm:username/string()) let $base := concat($zotero:API_BASE, "/groups/", $zotero:GROUP_ID, "/items") let $qs := string-join(( @@ -214,6 +216,7 @@ declare function zotero:sync($request as map(*)) { else () ), "&") let $href := concat($base, "?", $qs) + let $log := util:log('info','HREF ' || $href) let $req := @@ -310,7 +313,8 @@ declare function zotero:items-suggest($request as map(*)) { return map{ "key": string($i/@key), "title": string($i/title), - "bib": if ($i/bib/@html = "true") then string($i/bib) else "" + "bib": if ($i/bib/@html = "true") then string($i/bib) else "", + "tag": data($i//tag[1]) } } return serialize($arr, map{ "method":"json", "indent": true() }) diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index 1fbc46d..98d99eb 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -15,6 +15,7 @@ class ZoteroAutocomplete extends HTMLElement { this._debounceMs = 250; this._minlen = 2; this._uidBase = Math.random().toString(36).slice(2); + this._value = ''; // ← ALWAYS the TAG that Fore should get // light DOM markup this.classList.add('za'); @@ -105,30 +106,38 @@ class ZoteroAutocomplete extends HTMLElement { /* ===== Fore contract ===== */ get value() { - return this.$input?.dataset.key || ''; + // VALUE IS THE TAG (not the visible text) + return this._value || ''; } set value(v) { - const key = String(v || '').trim(); - if (!key) { + // v is TAG + const tag = String(v || '').trim(); + if (!tag) { this.clear(); this._notifyValueChanged(); return; } - this.$input.dataset.key = key; - this._selected = { key, title: '', bib: '' }; - this._fetchBib(key) + // store tag as the component value + this._value = tag; + this.$input.dataset.tag = tag; // keep for debugging/inspection + + this._selected = { tag, title: '', bib: '' }; + + // resolve bib using local cache first, then single request by tag + this._fetchBib(tag) .then(html => { this._selected.bib = html || ''; const plain = this._stripHtml(html || '') || ''; - this.$input.value = plain; - this._showOverlay(html || plain); + // show something human-friendly; we only know the tag here + this.$input.value = plain || tag; + this._showOverlay(html || plain || tag); this._toggleClear(); this._notifyValueChanged(); }) .catch(() => { - this.$input.value = key; - this._showOverlay(key); + this.$input.value = tag; + this._showOverlay(tag); this._toggleClear(); this._notifyValueChanged(); }); @@ -148,16 +157,10 @@ class ZoteroAutocomplete extends HTMLElement { const opts = { bubbles: true, composed: true }; const withDetail = name => new CustomEvent(name, { ...opts, detail: { value: val } }); - // fire on host + // fire on host ONLY (avoid inner input events that would expose human text) this.dispatchEvent(new Event('input', opts)); this.dispatchEvent(new Event('change', opts)); this.dispatchEvent(withDetail('value-changed')); - // and on inner input (compat) - if (this.$input) { - this.$input.dispatchEvent(new Event('input', opts)); - this.$input.dispatchEvent(new Event('change', opts)); - this.$input.dispatchEvent(withDetail('value-changed')); - } } /* ===== public helpers ===== */ @@ -165,8 +168,9 @@ class ZoteroAutocomplete extends HTMLElement { return this._selected || null; } clear() { + this._value = ''; this.$input.value = ''; - this.$input.dataset.key = ''; + this.$input.dataset.tag = ''; this._selected = null; this._render([]); this._hideOverlay(); @@ -183,7 +187,8 @@ class ZoteroAutocomplete extends HTMLElement { // real typing → drop selection & overlay and notify empty value if (this._selected) { this._selected = null; - this.$input.dataset.key = ''; + this._value = ''; + this.$input.dataset.tag = ''; this._hideOverlay(); this._notifyValueChanged(); } @@ -202,19 +207,42 @@ class ZoteroAutocomplete extends HTMLElement { const res = await fetch(url.toString(), { credentials: 'include' }); if (!res.ok) throw new Error('HTTP ' + res.status); const data = await res.json(); + + // Robust mapping: pull tag from several possible shapes + const firstTag = obj => { + if (!obj) return ''; + if (typeof obj.tag === 'string') return obj.tag; + if (Array.isArray(obj.tags) && obj.tags.length && typeof obj.tags[0]?.tag === 'string') + return obj.tags[0].tag; + if ( + obj.data && + Array.isArray(obj.data.tags) && + obj.data.tags.length && + typeof obj.data.tags[0]?.tag === 'string' + ) + return obj.data.tags[0].tag; + if (typeof obj.topTag === 'string') return obj.topTag; + if (typeof obj.matchTag === 'string') return obj.matchTag; + return ''; + }; + let list = (Array.isArray(data) ? data : data.items || []) .map(it => ({ key: it.key || it.data?.key || '', + tag: it.tag || firstTag(it) || '', title: it.title || it.data?.title || '', bib: it.bib || it.html || '', })) - .filter(x => x.key); + // keep items that have at least a tag (we need tag for value) + .filter(x => x.tag); - // If no bib included, fetch snippets for visible set + // If no bib included, fetch snippets for visible set (by TAG) if (list.length && !list[0].bib) { const limited = list.slice(0, this._limit); - const htmls = await Promise.all(limited.map(i => this._fetchBib(i.key).catch(() => ''))); - limited.forEach((i, idx) => (i.bib = htmls[idx] || this._escape(i.title || '[untitled]'))); + const htmls = await Promise.all( + limited.map(i => (i.tag ? this._fetchBib(i.tag).catch(() => '') : Promise.resolve(''))), + ); + limited.forEach((i, idx) => (i.bib = htmls[idx] || this._escape(i.title || i.tag || '[untitled]'))); list = limited; } this._render(list); @@ -224,33 +252,30 @@ class ZoteroAutocomplete extends HTMLElement { } } - // NEW: local helper to read bib from already loaded results (no network) - // NEW: local helper to read bib from already loaded results (no network) + // Local helper: read bib from current rendered items (no network) _getBib(id) { const needle = (id || '').trim().toLowerCase(); if (!needle) return ''; - const items = Array.isArray(this._results) ? this._results : []; + const items = Array.isArray(this._items) ? this._items : []; const hit = items.find(it => { const tag = (it && it.tag ? String(it.tag) : '').toLowerCase(); const key = (it && it.key ? String(it.key) : '').toLowerCase(); - return tag === needle || key === needle; + return (tag && tag === needle) || (key && key === needle); }); + return hit && typeof hit.bib === 'string' ? hit.bib : ''; } - // REPLACE your current _fetchBib with this version + // Use cached suggestions first; if not found (preset value), resolve once by TAG async _fetchBib(id) { - // 1) try already-loaded suggestions const local = this._getBib(id); if (local) return local; - // 2) fallback for preset values: resolve one item by tag const tag = (id || '').trim(); if (!tag) return ''; try { - // keep your existing endpoint; resolve relative to the current document URL const url = new URL(this._endpoint, document.baseURI); url.searchParams.set('tag', tag); url.searchParams.set('limit', '1'); @@ -286,9 +311,9 @@ class ZoteroAutocomplete extends HTMLElement { li.id = id; li.setAttribute('role', 'option'); li.setAttribute('data-idx', String(idx)); - li.setAttribute('data-key', it.key); + li.setAttribute('data-tag', it.tag || ''); li.tabIndex = -1; - li.innerHTML = `
    ${it.bib || this._escape(it.title || '[untitled]')}
    `; + li.innerHTML = `
    ${it.bib || this._escape(it.title || it.tag || '[untitled]')}
    `; li.addEventListener('mouseenter', () => this._setActive(idx, true)); this.$list.appendChild(li); }); @@ -372,14 +397,6 @@ class ZoteroAutocomplete extends HTMLElement { const related = e.relatedTarget; if (!this.contains(related)) this._render([]); } - /* _handleFocus() { - const q = this.$input.value.trim(); - if (q.length >= this._minlen && this._items.length) { - this.$list.classList.add('is-open'); - this.$input.setAttribute('aria-expanded', 'true'); - } - this._toggleClear(); - } */ _handleFocus() { // if overlay is active, ensure no text selection band is visible if (this.$field?.classList.contains('has-overlay')) { @@ -439,19 +456,26 @@ class ZoteroAutocomplete extends HTMLElement { _choose(idx) { const item = this._items[idx]; if (!item) return; + + // Store selection this._selected = item; - const plain = this._stripHtml(item.bib) || item.title || ''; + const plain = this._stripHtml(item.bib) || item.title || item.tag || ''; this.$input.value = plain; - this.$input.dataset.key = item.key; + + // VALUE MUST BE TAG (not key, not title) + this._value = item.tag || ''; + this.$input.dataset.tag = this._value; + this._render([]); // hide menu this._showOverlay(item.bib || plain); this._toggleClear(); - this._notifyValueChanged(); // Fore: emit + // Fore: emit (value == tag) + this._notifyValueChanged(); this.dispatchEvent( new CustomEvent('zotero-select', { bubbles: true, - detail: { key: item.key, title: item.title || '', bib: item.bib || '' }, + detail: { key: item.key, tag: item.tag, title: item.title || '', bib: item.bib || '' }, }), ); } From e03f8a51f7e6672617726dd04262baf795a7d6af Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Thu, 16 Oct 2025 17:48:01 +0200 Subject: [PATCH 098/254] fixes + integration in bibliography.html --- src/resources/scripts/zotero-autocomplete.js | 135 ++++++++++++++++--- src/templates/edit.html | 3 + src/templates/parts/bibliography.html | 13 +- 3 files changed, 129 insertions(+), 22 deletions(-) diff --git a/src/resources/scripts/zotero-autocomplete.js b/src/resources/scripts/zotero-autocomplete.js index 98d99eb..bd2500d 100644 --- a/src/resources/scripts/zotero-autocomplete.js +++ b/src/resources/scripts/zotero-autocomplete.js @@ -3,19 +3,24 @@ */ class ZoteroAutocomplete extends HTMLElement { static get observedAttributes() { - return ['endpoint', 'bib-endpoint', 'tag', 'limit', 'minlength', 'debounce', 'value', 'name']; + return ['endpoint', 'tag', 'limit', 'debounce', 'value', 'name', 'wait-ready']; } constructor() { super(); // state + this.fore = {}; this._items = []; this._active = -1; this._selected = null; this._debounceMs = 250; this._minlen = 2; this._uidBase = Math.random().toString(36).slice(2); - this._value = ''; // ← ALWAYS the TAG that Fore should get + this._value = ''; // Fore-facing value (TAG) + this._deferUntilReady = this.hasAttribute('wait-ready'); + this._ready = !this._deferUntilReady; + this._pendingValue = null; // value set before ready is fired + this._readyHandler = null; // light DOM markup this.classList.add('za'); @@ -50,12 +55,14 @@ class ZoteroAutocomplete extends HTMLElement { } connectedCallback() { - // config + // config (initial — may be template, re-read on 'ready' if deferred) this._endpoint = this.getAttribute('endpoint') || '/api/zotero/items/suggest'; this._tag = this.getAttribute('tag') || ''; this._limit = parseInt(this.getAttribute('limit') || '8', 10); this._minlen = parseInt(this.getAttribute('minlength') || String(this._minlen), 10); this._debounceMs = parseInt(this.getAttribute('debounce') || String(this._debounceMs), 10); + this._deferUntilReady = this.hasAttribute('wait-ready'); + this._ready = !this._deferUntilReady; // listeners this.$input.addEventListener('input', this._onInput); @@ -70,12 +77,27 @@ class ZoteroAutocomplete extends HTMLElement { const nameAttr = this.getAttribute('name'); if (nameAttr) this.$input.name = nameAttr; - // ensure at least one notification after connect - queueMicrotask(() => this._notifyValueChanged()); - - // preset value via attribute (fires notifications as well) - const initVal = this.getAttribute('value'); - if (initVal) this.value = initVal; + if (this._deferUntilReady) { + // wait for Fore to resolve template attributes & models + this._readyHandler = () => this._onReadyOnce(); + document.addEventListener('ready', this._readyHandler, { once: true }); + // document.addEventListener('model-construct-done', this._readyHandler, { once: true }); + + // If a preset value is already present, stash it so we can resolve after 'ready' + const initVal = this.getAttribute('value'); + if (initVal) { + this._pendingValue = String(initVal).trim(); + // provide a harmless placeholder so users see *something* before ready + this.$input.value = this._pendingValue; + this._showOverlay(this._pendingValue); + this._toggleClear(); + } + } else { + // normal immediate init + queueMicrotask(() => this._notifyValueChanged()); + const initVal = this.getAttribute('value'); + if (initVal) this.value = initVal; + } } disconnectedCallback() { @@ -86,6 +108,11 @@ class ZoteroAutocomplete extends HTMLElement { this.removeEventListener('focusout', this._onBlur); this.removeEventListener('focusin', this._onFocus); this.$clear.removeEventListener('click', this._onClear); + if (this._readyHandler) { + document.removeEventListener('ready', this._readyHandler); + // document.removeEventListener('model-construct-done', this._readyHandler); + this._readyHandler = null; + } } attributeChangedCallback(name, _old, value) { @@ -101,7 +128,47 @@ class ZoteroAutocomplete extends HTMLElement { this.$input?.addEventListener('input', this._onInput); } if (name === 'name' && value) this.$input.name = value; - if (name === 'value' && value !== this.value) this.value = value || ''; + if (name === 'wait-ready') this._deferUntilReady = this.hasAttribute('wait-ready'); + + if (name === 'value' && value !== this.value) { + // If deferring, just stash it until ready + if (this._deferUntilReady && !this._ready) { + this._pendingValue = String(value || '').trim(); + // lightweight placeholder (no network before ready) + this.$input.value = this._pendingValue; + this._showOverlay(this._pendingValue); + this._toggleClear(); + } else { + this.value = value || ''; + } + } + } + + // Called once when Fore emits 'ready' (or 'model-construct-done') + _onReadyOnce() { + this._ready = true; + // re-read endpoint after template resolution + this._endpoint = this.getAttribute('endpoint') || this._endpoint; + + // Prefer pending value set via property/attribute before ready; else attribute value + const start = + (this._pendingValue && this._pendingValue.trim()) || + (this.getAttribute('value') && this.getAttribute('value').trim()) || + ''; + + if (start) { + // Apply now with full resolution (fetch bib if needed) + this._pendingValue = null; + this.value = start; + } else { + this._notifyValueChanged(); + } + // cleanup listeners + if (this._readyHandler) { + document.removeEventListener('ready', this._readyHandler); + document.removeEventListener('model-construct-done', this._readyHandler); + this._readyHandler = null; + } } /* ===== Fore contract ===== */ @@ -118,9 +185,21 @@ class ZoteroAutocomplete extends HTMLElement { return; } - // store tag as the component value + // If we’re deferring and not ready yet, just stage and show placeholder (no fetch) + if (this._deferUntilReady && !this._ready) { + this._pendingValue = tag; + this._value = tag; + this.$input.dataset.tag = tag; + this.$input.value = tag; + this._showOverlay(tag); + this._toggleClear(); + // Do not notify yet; Fore will sync on ready + return; + } + + // store tag as the component value (ready path) this._value = tag; - this.$input.dataset.tag = tag; // keep for debugging/inspection + this.$input.dataset.tag = tag; this._selected = { tag, title: '', bib: '' }; @@ -132,12 +211,14 @@ class ZoteroAutocomplete extends HTMLElement { // show something human-friendly; we only know the tag here this.$input.value = plain || tag; this._showOverlay(html || plain || tag); + this._setTagHints(tag); this._toggleClear(); this._notifyValueChanged(); }) .catch(() => { this.$input.value = tag; this._showOverlay(tag); + this._setTagHints(tag); this._toggleClear(); this._notifyValueChanged(); }); @@ -157,7 +238,7 @@ class ZoteroAutocomplete extends HTMLElement { const opts = { bubbles: true, composed: true }; const withDetail = name => new CustomEvent(name, { ...opts, detail: { value: val } }); - // fire on host ONLY (avoid inner input events that would expose human text) + // fire on host ONLY (avoid inner input events that expose human text) this.dispatchEvent(new Event('input', opts)); this.dispatchEvent(new Event('change', opts)); this.dispatchEvent(withDetail('value-changed')); @@ -169,6 +250,7 @@ class ZoteroAutocomplete extends HTMLElement { } clear() { this._value = ''; + this._pendingValue = null; this.$input.value = ''; this.$input.dataset.tag = ''; this._selected = null; @@ -177,6 +259,23 @@ class ZoteroAutocomplete extends HTMLElement { this._toggleClear(); } + _setTagHints(tag) { + const t = tag && String(tag).trim() ? `tag: ${tag.trim()}` : ''; + if (this.$overlay) { + if (t) { + this.$overlay.setAttribute('title', t); + this.$overlay.setAttribute('aria-label', t); + } else { + this.$overlay.removeAttribute('title'); + this.$overlay.removeAttribute('aria-label'); + } + } + if (this.$input) { + if (t) this.$input.setAttribute('title', t); + else this.$input.removeAttribute('title'); + } + } + /* ===== input/search ===== */ async _handleInput(e) { // ignore synthetic input events (Fore/programmatic) to prevent overlay flicker @@ -229,11 +328,10 @@ class ZoteroAutocomplete extends HTMLElement { let list = (Array.isArray(data) ? data : data.items || []) .map(it => ({ key: it.key || it.data?.key || '', - tag: it.tag || firstTag(it) || '', + tag: (it.tag || firstTag(it) || '').trim(), title: it.title || it.data?.title || '', bib: it.bib || it.html || '', })) - // keep items that have at least a tag (we need tag for value) .filter(x => x.tag); // If no bib included, fetch snippets for visible set (by TAG) @@ -259,8 +357,8 @@ class ZoteroAutocomplete extends HTMLElement { const items = Array.isArray(this._items) ? this._items : []; const hit = items.find(it => { - const tag = (it && it.tag ? String(it.tag) : '').toLowerCase(); - const key = (it && it.key ? String(it.key) : '').toLowerCase(); + const tag = (it && it.tag ? String(it.tag) : '').trim().toLowerCase(); + const key = (it && it.key ? String(it.key) : '').trim().toLowerCase(); return (tag && tag === needle) || (key && key === needle); }); @@ -463,11 +561,12 @@ class ZoteroAutocomplete extends HTMLElement { this.$input.value = plain; // VALUE MUST BE TAG (not key, not title) - this._value = item.tag || ''; + this._value = (item.tag || '').trim(); this.$input.dataset.tag = this._value; this._render([]); // hide menu this._showOverlay(item.bib || plain); + this._setTagHints(this._value); this._toggleClear(); // Fore: emit (value == tag) diff --git a/src/templates/edit.html b/src/templates/edit.html index 3d79527..80fef5f 100644 --- a/src/templates/edit.html +++ b/src/templates/edit.html @@ -15,12 +15,14 @@ + + @@ -122,6 +124,7 @@ + ${app} hidden false diff --git a/src/templates/parts/bibliography.html b/src/templates/parts/bibliography.html index f437c03..77e3318 100644 --- a/src/templates/parts/bibliography.html +++ b/src/templates/parts/bibliography.html @@ -57,13 +57,18 @@

    - + - + +
    From 996f56ce591d78a21d2fa99f4b76a8bdb744cfc1 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 17 Oct 2025 10:35:04 +0200 Subject: [PATCH 101/254] bib endpoint handler deleted (not needed any more) --- src/modules/lib/api/zotero.xql | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/src/modules/lib/api/zotero.xql b/src/modules/lib/api/zotero.xql index 7c21ad4..94b0ab3 100644 --- a/src/modules/lib/api/zotero.xql +++ b/src/modules/lib/api/zotero.xql @@ -369,37 +369,3 @@ declare %private function zotero:_load-json($key as xs:string) as map(*)? { return if ($txt ne "") then try { parse-json($txt) } catch * { () } else () }; -declare function zotero:item-bib($request as map(*)) { - response:set-header("Content-Type", "text/html; charset=UTF-8"), - - let $key := normalize-space(request:get-parameter("key", "")) - let $tag := lower-case(normalize-space(request:get-parameter("tag", ""))) - let $emit := function($html as xs:string) as empty-sequence() { - response:stream-binary(util:string-to-binary($html), "text/html", ()), () - } - - return - if ($key ne "") then - let $x := doc(concat($config:zotero-items-xml-dir, "/", $key, ".xml"))/item - return - if (empty($x)) then ( response:set-status-code(404), $emit("") ) - else - let $bib := if ($x/bib/@html="true") then string($x/bib) else "" - return - if ($bib ne "") then $emit($bib) - else - let $json := zotero:_load-json($key) - let $title := string(($json?data?title, "")[1]) - return $emit(zotero:_fallback-html($title)) - else if ($tag ne "") then - let $x := (collection($config:zotero-items-xml-dir)/item[normalize-space(@parentItem) = "" and tags/tag = $tag])[1] - return - if (empty($x)) then ( response:set-status-code(404), $emit("") ) - else - let $bib := if ($x/bib/@html="true") then string($x/bib) else "" - return - if ($bib ne "") then $emit($bib) - else $emit(zotero:_fallback-html(string($x/title))) - else - ( response:set-status-code(400), $emit("") ) -}; From e36fc64bc66f9e4868b034b25f43a67185b1eff0 Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 17 Oct 2025 13:03:50 +0200 Subject: [PATCH 102/254] zotero sync page to trigger from client --- src/templates/zotero-sync.html | 132 ++++++++++++++++++++++ src/templates/zotero.html | 200 --------------------------------- 2 files changed, 132 insertions(+), 200 deletions(-) create mode 100644 src/templates/zotero-sync.html delete mode 100644 src/templates/zotero.html diff --git a/src/templates/zotero-sync.html b/src/templates/zotero-sync.html new file mode 100644 index 0000000..72d7076 --- /dev/null +++ b/src/templates/zotero-sync.html @@ -0,0 +1,132 @@ + + + + + Fore: Trigger Button with Spinner + + + + + + + + +
    + + + + + { + "updated": "-", + "libraryVersion": "-" + } + + + + notrunning + + + + + + + + + notrunning + + + + notrunning + + + + + +

    Zotero Sync Page

    + + + + + + + + + running + + + + + + +
    +
    updated items: {?updated}
    +
    library version: {?libraryVersion}
    +
    +     +
    + + diff --git a/src/templates/zotero.html b/src/templates/zotero.html deleted file mode 100644 index a9d0b29..0000000 --- a/src/templates/zotero.html +++ /dev/null @@ -1,200 +0,0 @@ - - - - - - - zotero - - - - - - - - -
    - - - - - - - - - - - - [{}] - - - - - - - - - - - - false - - - - - - - - - - - - loading items - - -

    Zotero Test Page

    -
    -
    - -

    Synchronize from Zotero

    - - - -

    Search

    -
    - - - - - open - - - busy - - - - - - - - - -
    - - - - - - - - - - - - - - - - -
    - -
    - -

    Get a bib

    - - - - -
    - -
    - - - - - - - - -
    -     -
    - - From 706d7cf47931e3669c52bdbfb36e379b9276ad6b Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 17 Oct 2025 13:04:08 +0200 Subject: [PATCH 103/254] moved sync endpoint back to custom-api.json --- src/modules/custom-api.json | 18 ++++++++++++++++++ src/modules/lib/api.json | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/modules/custom-api.json b/src/modules/custom-api.json index a1c36b5..7acf413 100644 --- a/src/modules/custom-api.json +++ b/src/modules/custom-api.json @@ -850,6 +850,24 @@ } } }, + "/api/zotero/sync": { + "get": { + "summary": "Incrementally sync local cache.", + "tags": ["zotero"], + "operationId": "zotero:sync", + "responses": { + "200": { + "description": "Sync result.", + "content": { + "text/plain": { "schema": { "type": "string" } } + } + }, + "304": { + "description": "No updates." + } + } + } + }, "/api/zotero/items/suggest": { "get": { "security": [{ "basicAuth": [] }], diff --git a/src/modules/lib/api.json b/src/modules/lib/api.json index a547447..1ded04c 100644 --- a/src/modules/lib/api.json +++ b/src/modules/lib/api.json @@ -2845,24 +2845,6 @@ } } }, - "/api/zotero/sync": { - "get": { - "summary": "Incrementally sync local cache.", - "tags": ["zotero"], - "operationId": "zotero:sync", - "responses": { - "200": { - "description": "Sync result.", - "content": { - "text/plain": { "schema": { "type": "string" } } - } - }, - "304": { - "description": "No updates." - } - } - } - }, "/api/register/{type}/{id}" : { "post": { "summary": "Create a local entry", From 8f7418fd1a8676e0d4b2dadf0a41f5068fb0a67e Mon Sep 17 00:00:00 2001 From: Joern Turner Date: Fri, 17 Oct 2025 13:30:12 +0200 Subject: [PATCH 104/254] changed to use new component --- src/templates/parts/bibliography.html | 2 +- src/templates/parts/images.html | 11 ++++++++--- src/templates/parts/prev-editions.html | 11 ++++++++--- src/templates/parts/transmission.html | 11 ++++++++--- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/templates/parts/bibliography.html b/src/templates/parts/bibliography.html index 77e3318..167bc72 100644 --- a/src/templates/parts/bibliography.html +++ b/src/templates/parts/bibliography.html @@ -57,7 +57,7 @@
    - + diff --git a/src/templates/parts/images.html b/src/templates/parts/images.html index 8210df2..4b9e0fc 100644 --- a/src/templates/parts/images.html +++ b/src/templates/parts/images.html @@ -64,9 +64,14 @@ - + +