WIP: Altered data model

.devcontainer/Dockerfile.dev

@@ -3,7 +3,7 @@ FROM mcr.microsoft.com/devcontainers/java:8-bullseye
 ARG GITLAB_TOKEN=""
 ARG EXIST_VERSION=6.2.0
 ARG TEMPLATING_VERSION=1.1.0
-ARG PUBLISHER_LIB_VERSION=4.0.0
+ARG PUBLISHER_LIB_VERSION=6.0.1
 ARG ROUTER_VERSION=1.8.0
 ARG EDITOR_VERSION=1.1.0
 

.existdb.json

@@ -17,7 +17,9 @@
       "node_modules/**",
       "bower_components/**",
       "build/**",
-      "webtest/**"
+      "webtest/**",
+      ".idea/**",
+      ".DS_Store"
     ]
   },
   "package": {

.github/workflows/e2e.yml

@@ -0,0 +1,124 @@
+# This workflow builds a xar archive, deploys it into exist and execute a simple smoke test.
+
+name: Build and Test
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    strategy:
+       fail-fast: false
+       matrix:
+        # 7.0.0-SNAPSHOT and 6.4.0 created
+         exist-version: [6.4.0, release]
+         java-version: [11]
+         experimental: [false]
+         include:
+           - exist-version: latest
+             java-version: 21
+             experimental: true
+
+    steps:
+      # Checkout code
+      - uses: actions/checkout@v5
+
+      # Speed up apt-get installs
+      - name: Disable man-db to make package install and removal faster
+        run: |
+          echo 'set man-db/auto-update false' | sudo debconf-communicate >/dev/null
+          sudo dpkg-reconfigure man-db
+
+      - name: Install Test Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libxml2-utils bats
+          npm install --global @existdb/xst
+
+      # sanity check
+      - name: Ensure all XML files are well-formed
+        run: |
+          xmllint --noout \
+            $(find . -type f -name '*.xml')
+
+      # Build
+      - name: Install Java
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java-version }}
+      - run: ant
+
+      - name: Check Contents
+        working-directory: build/
+        run: ls
+
+      # Install
+      - name: Start exist-ci containers
+        run: |
+          docker run -dit -p 8080:8080 \
+          --name exist --rm --health-interval=1s --health-start-period=2s \
+          duncdrum/existdb:${{ matrix.exist-version }}-slim
+
+      - name: Wait for server to start
+        timeout-minutes: 1
+        run: |
+          echo "Waiting for server to start..."
+          until docker logs exist 2>&1 | grep -q "Server has started"; do
+            echo "Still waiting..."
+            sleep 4
+          done
+
+      - name: install edep and edep-data
+        env:
+          EXISTDB_USER: 'admin'
+          EXISTDB_PASS: ''
+          EXISTDB_SERVER: 'http://localhost:8080/'
+        run: |
+          xst package install ./dist/*.xar
+          xst package install github-release edep-data --owner eeditiones
+
+      # Test
+      - name: Run smoke test
+        run: bats --tap test/*.bats
+
+      - name: debug logs
+        if: failure()
+        run: docker logs exist | grep 'ERROR'
+
+      - name: Copy logs on failure
+        if: failure()
+        run: |
+          echo "Copying logs..."
+          docker cp exist:/exist/logs/exist.log ./exist.log
+
+      - name: Upload logs artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: edep-exist-${{ matrix.exist-version }}-log
+          path: exist.log
+
+      - name: Run Cypress e2e test
+        if: matrix.experimental != true
+        uses: cypress-io/github-action@v6
+        with:
+          browser: firefox
+
+      - name: Run Cypress e2e test (expect fail)
+        if: matrix.experimental == true
+        continue-on-error: true
+        uses: cypress-io/github-action@v6
+        with:
+          browser: firefox
+
+      - name: Upload cypress sceenshots
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: CiM-exist-${{ matrix.exist-version }}-screenshots
+          path: cypress/screenshots/

.github/workflows/linting.yml

@@ -0,0 +1,11 @@
+name: Linting
+on: push
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install modules
+      run: npm i
+    - name: Run ESLint and Prettier
+      run: npm run lint

.gitignore

@@ -1,4 +1,5 @@
-.idea
+.idea/**
 build/
 dist/
-node_modules/
+node_modules/
+.DS_Store

.prettierrc.mjs

@@ -0,0 +1,12 @@
+/**
+ * @type {import('prettier').Config}
+ */
+const config = {
+    singleQuote: true,
+    arrowParens: 'avoid',
+    tabWidth: 4,
+    printWidth: 120,
+    bracketSameLine: true,
+};
+
+export default config;

Dockerfile

@@ -35,7 +35,7 @@ ENV PATH ${PATH}:${ANT_HOME}/bin
 FROM builder as tei
 
 ARG TEMPLATING_VERSION=1.1.0
-ARG PUBLISHER_LIB_VERSION=4.0.0
+ARG PUBLISHER_LIB_VERSION=6.0.1
 ARG ROUTER_VERSION=1.8.1
 
 # add key

README.md

@@ -8,9 +8,11 @@ EDEp implements an editor and toolbox for epigraphic data adhering to the EpiDoc
 * web components for transcribing inscriptions using Leiden markup
 * components allowing controlled XML editing for specific micro-environments within the larger form
 * ODDS based on the TEI Processing Model for output transformations: rather than using hand-written transformations we describe them in a declarative way in TEI itself
-* a component for looking up bibliographic references in Zotero
+* API for synchronizing Zotero library to the local instance
+* a component for looking up bibliographic references in locally cached Zotero
 
-The current state of the application itself should be considered an **early beta**, i.e. fully usable, but still under development.
+The current state of the application itself should be considered an **early beta**, i.e. usable, but still under development.
+Please report any hiccups via the issue tracker.
 
 ## Installation
 
@@ -19,7 +21,7 @@ You may either
 1. install the application into an existing eXist instance or
 2. use the docker image
 
-Using option 1, the application requires at least Java version 8 and eXist version 6.2.0. Details of the installation process are described in the [TEI Publisher documentation](https://tei-publisher.com/exist/apps/tei-publisher/documentation/exist-installation). You may skip the last step: *Installing TEI Publisher*, which is not needed for EDEp, but open the dashboard as described. Next, download the EDEp application and data package from the [GitHub release page](https://github.com/eeditiones/edep/releases). Install those one after the other: either click on the *Upload* button and select the downloaded `edep-*.xar` or drag and drop it onto the button.
+Using option 1, the application requires at least Java version 8 and eXist version 6.2.0. Details of the installation process are described in the [TEI Publisher documentation](https://tei-publisher.com/exist/apps/tei-publisher/documentation/exist-installation). You may skip the last step: *Installing TEI Publisher*, which is not needed for EDEp, but open the dashboard as described. Next, download the EDEp application and data package from the [GitHub release page](https://github.com/eeditiones/edep/releases). Install those one after the other: first the data package, then the application itself. Either click on the *Upload* button and select the downloaded `edep-*.xar` or drag and drop it onto the button.
 
 Option 2 is **recommended** for new users and easier if you would like a working environment without having to install Java and eXist. You need docker installed though. Windows and Mac users may download the [docker desktop](https://www.docker.com/products/docker-desktop) application. After install, open a shell and run the following pull command:
 
@@ -49,6 +51,14 @@ Log in by clicking the *login* button in the top right corner of the page. The d
 
 ![demo collection](doc/edep-demo.png)
 
+## Zotero synchronization
+
+Use the `api.html` to manually trigger Zotero synchronization. It usually takes 2-3 minutes the first time, couple of seconds afterwards.
+
+![zotero-sync](doc/zotero.png)
+
+Read more on this subject in [Zotero cache documentation](doc/zotero-cache.md)
+
 ## Customization
 
 **Important note**: just like TEI, EpiDoc is **not** a standard in the sense that there's only one way to encode things. It does not strictly standardize every little detail, but rather tries to provide a common base. Many aspects depend on the concrete research object and will necessarily differ. Therefore you cannot expect that the editor will consume every possible variation of EpiDoc out of the box. It should handle the common denominator, i.e. common parts like titles, transcriptions, translations or commentaries, but may fail to correctly interpret other details of your encoding, which require further customization.
@@ -60,37 +70,41 @@ Therefore the editor works entirely within the web browser: it only loads the Ep
 For example, the form field for the state of preservation is defined in the HTML as follows:
 
 ```html
-<fx-control ref="physDesc/objectDesc/supportDesc/condition" update-event="input">
+<fx-control ref="objectDesc/supportDesc/condition" update-event="input">
     <label>
-        <pb-i18n key="form.preservation">Erhaltung</pb-i18n>
+        <pb-i18n key="form.preservation"/>
     </label>
-    <select class="widget">
-        <option value="" data-i18n="form.unknown">unbekannt</option>
-        <option value="complete" data-i18n="preservation.complete">vollständig</option>
-        <option value="fragmentary" data-i18n="preservation.fragmentary">fragmentarisch</option>
-    </select>
+    <select
+        ref="instance('preservation')//category"
+        class="widget">
+        <template>
+            <option value="{@xml:id}">{catDesc[@xml:lang = $lang]}</option>
+        </template>
+    </select>     
 </fx-control>
 ```
 
 HTML elements starting with `fx-` belong to *Fore*, the forms framework. *Fore* was designed to preserve the good parts of the – now outdated – XForms standard and takes them to the next level. As with all powerful tools, it requires a bit of time to fully grasp the concepts and understand how they work together. We suggest to read the [article series](https://medium.com/@joern.turner/fore-elements-explained-part-1-89fc41ec6923), which explains the available elements.
 
-The `@ref` attribute binds the form control to the element selected by the given XPath expression, in this case: `physDesc/objectDesc/supportDesc/condition` within the outer `msPart`. All labels are multi-lingual (German and English) by default, which is why we use the `<pb-i18n>` tag and `data-i18n` attributes to reference translation keys. Translations for all labels can be found in [src/resources/i18n/app](src/resources/i18n/app).
+The `@ref` attribute binds the form control to the element selected by the given XPath expression, in this case: `objectDesc/supportDesc/condition`. All labels are multi-lingual (German and English) by default, which is why we use the `<pb-i18n>` tag and to reference translation keys. Translations for all labels can be found in [src/resources/i18n/app](src/resources/i18n/app).
+
+Select options labels are also localized, but since these are read from TEI taxonomies, the lookups search for `catDesc` elements with an `xml:lang` attribute matching the current language (`$lang` in the example above).
 
-The design decision to use `msPart` was made to support inscriptions consisting of multiple fragments on the same object. EDEp therefore distinguishes between properties which apply to the object as a whole and those specific to the inscription. Consequently every EpiDoc document in EDEp has at least one `msPart` for the main inscription. Note however, that support for multiple fragments is still a bit lacking and needs more work.
+When creating a new document, the editor starts by loading an empty [EpiDoc template](src/templates/fore/epidoc-template.xml). This contains placeholders for all the elements covered by the form. Some elements, e.g. bibliographic entries, are repeatable, which means you can add more of the same type. Sub-templates for repeat elements and other fragments inserted on demand are located in [templates.xml](src/templates/fore/templates.xml).
 
-When creating a new document, the editor starts by loading an empty [EpiDoc template](src/templates/fore/epidoc-template.xml). This contains placeholders for all the elements covered by the form. Some elements, e.g. bibliographic entries, are repeatable, which means you can add more of the same type. Sub-templates for those elements are mostly located in [templates.xml](src/templates/fore/templates.xml).
+The entire pre-processing pipeline is started via `custom-api.xql/api:file-upload` which in turn call on other functions to convert the input.
 
-The form also needs some auxiliary data lists, e.g. for object types, materials and much more, which are mostly loaded from the data package.
+The form also needs some auxiliary data lists, e.g. for object types, materials and much more, which are loaded from `data/taxonomy` collection in the data package.
 
 To extend the form to include an additional element or attribute, one would proceed as follows:
 
-1. edit [epidoc-template.xml](src/templates/fore/epidoc-template.xml) and add the missing element or attribute. If it should be part of an `msPart` (i.e. target the inscription), make sure to also modify [mspart-tmpl.xml](src/templates/fore/mspart-tmpl.xml) in the same way. If the element or attribute belongs to a repeatable section, check [templates.xml](src/templates/fore/templates.xml).
-2. add a form control to [edit.html](src/templates/edit.html) and bind it to the element/attribute.
+1. edit [epidoc-template.xml](src/templates/fore/epidoc-template.xml) and add the missing element or attribute. If the element or attribute belongs to a repeatable section, adjust also [templates.xml](src/templates/fore/templates.xml).
+2. add a form control to [edit.html](src/templates/edit.html) or its dynamically imported form fragments and bind it to the element/attribute.
 
-Many fields in the form may contain inline TEI/XML. We have thus developed an XML editor component (based on the excellent codemirror library), which can be plugged into the form to replace any plain-text input field:
+Many fields in the form may contain inline TEI/XML. We have therefore developed an XML editor component (based on the codemirror library), which can be plugged into the form to replace any plain-text input field:
 
 ```html
-<fx-control class="commentary editor" ref="instance('default')//body/div[@type='commentary']" as="node" update-event="blur">
+<fx-control class="commentary editor" ref="div[@type='commentary']" as="node" update-event="blur">
     <jinn-xml-editor id="commentary" class="widget" unwrap="unwrap" placeholder="[Inline text/markup or sequence of paragraphs]" schema="resources/scripts/tei.json" schema-root="div" namespace="http://www.tei-c.org/ns/1.0">
         <div slot="toolbar">
             <pb-popover>
@@ -122,7 +136,7 @@ The editor features context-sensitive suggestions for elements and attributes wh
 
 ### Treatment of empty elements/attributes
 
-As explained above, the form works directly on the EpiDoc TEI XML. However, note that missing elements or attributes will be regarded as irrelevant by the form and the corresponding controls will be hidden. This is by design and conceptually an important feature! It is thus required that you **provide an empty placeholder** in the XML templates for every element or attribute to be edited in the form. For example, the template for the `<provenance>` element specifies empty attributes for `when` and `when-custom` even though those are mutually exclusive:
+As explained above, the form works directly on the EpiDoc TEI XML. However, real TEI documents will not necessarily contain all potentially possible elements and attributes. In such cases, note that "missing" elements or attributes will be regarded as irrelevant by the form and the corresponding controls will not be shown. This is an element of the Fore design and conceptually an important feature! It is therefore required that  **a placeholder** is provided in the XML templates for every element or attribute to be edited in the form or to indicate in the form control that a given attribute should be created on demand. For example, the template for the `<provenance>` element specifies empty attributes for `when` and `when-custom` even though those are mutually exclusive:
 
 ```xml
 <provenance type="found" when="" when-custom="" notBefore="" notAfter="">

cypress.config.js

@@ -0,0 +1,10 @@
+const { defineConfig } = require('cypress');
+
+module.exports = defineConfig({
+    e2e: {
+        baseUrl: 'http://localhost:8080/exist/apps/edep',
+        setupNodeEvents(on, config) {
+            // implement node event listeners here
+        },
+    },
+});

cypress/e2e/smoketest.cy.js

@@ -0,0 +1,17 @@
+describe('Smoketest', () => {
+    it('Can open the edit page', { defaultCommandTimeout: 10000 }, () => {
+        cy.visit('edit/demo/E0000028.xml');
+
+        cy.get('.edepid').should('have.value', 'E0000028');
+
+        // There are fragments here
+        cy.get('#r-fragments a').should('have.length', '2');
+
+        // We've seen this option list to break in some cases
+        cy.get('#findspot-ctrl option')
+            .should('have.length.of.at.least', 10)
+            .should('not.contain', '{.}')
+            .should('not.have.attr', 'value', '{@xml:id}')
+            .should('contain', 'A');
+    });
+});

cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "[email protected]",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })

cypress/support/e2e.js

@@ -0,0 +1,24 @@
+// ***********************************************************
+// This example support/e2e.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+Cypress.on('uncaught:exception', (err, runnable) => {
+    if (err.stack.includes('jinn-codemirror-bundle.js')) {
+        return false;
+    }
+    return true;
+});