Extending AI Actions

code_samples/ai_actions/assets/js/addAudioModule.js

@@ -0,0 +1,5 @@
+import { addModule } from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/create.ai.module';
+import { default as TranscribeAudio } from './transcribe.audio'
+
+addModule(TranscribeAudio );
+

code_samples/ai_actions/assets/js/transcribe.audio.js

@@ -0,0 +1,64 @@
+import BaseAIComponent from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/base.ai.component';
+
+export default class TranscribeAudio extends BaseAIComponent {
+    constructor(mainElement, config) {
+        super(mainElement, config);
+
+        this.requestHeaders = {
+            Accept: 'application/vnd.ibexa.api.ai.AudioText+json',
+            'Content-Type': 'application/vnd.ibexa.api.ai.TranscribeAudio+json',
+        };
+    }
+
+    getAudioInBase64() {
+        var request = new XMLHttpRequest();
+        request.open('GET', this.inputElement.href, false);
+        request.overrideMimeType('text\/plain; charset=x-user-defined');
+        request.send();
+
+        if (request.status === 200) {
+            return this.convertToBase64(request.responseText);
+        }
+    }
+
+    getRequestBody() {
+        const body = {
+            TranscribeAudio: {
+                Audio: {
+                    base64: this.getAudioInBase64(),
+                },
+                RuntimeContext: {},
+            },
+        };
+
+        if (this.languageCode) {
+            body.TranscribeAudio.RuntimeContext.languageCode = this.languageCode;
+        }
+
+        return JSON.stringify(body);
+    }
+
+    afterFetchData(response) {
+        super.afterFetchData();
+
+        if (response) {
+            this.outputElement.value = response.AudioText.Text.text[0];
+        }
+    }
+
+    toggle(forceEnabled) {
+        super.toggle(forceEnabled);
+
+        this.outputElement.disabled = !forceEnabled || !this.outputElement.disabled;
+    }
+
+    convertToBase64(data) {
+        var binary = ""
+
+        for(var i=0;i<data.length;i++){
+            binary += String.fromCharCode(data.charCodeAt(i) & 0xff);
+        }
+
+        return btoa(binary);
+    }
+}

code_samples/ai_actions/config/packages/ibexa_admin_ui.yaml

@@ -0,0 +1,88 @@
+parameters:
+    # Admin siteaccess group name
+    admin_group_name: admin_group
+
+ibexa:
+    siteaccess:
+        list: [admin]
+        groups:
+            # WARNING: Do not remove or rename this group.
+            # It's used to distinguish common siteaccesses from admin ones.
+            # In case of multisite with multiple admin panels, remember to add any additional admin siteaccess to this group.
+            admin_group: [admin]
+
+    system:
+        admin_group:
+            # System languages. Note that by default, content, content types, and other data are in eng-GB locale,
+            # so removing eng-GB from this list may lead to errors or content not being shown, unless you change
+            # all eng-GB data to other locales first.
+            # For admin this needs to contain all languages you want to translate content to on the given repository.
+            languages: [eng-GB]
+            content_tree_module:
+                contextual_tree_root_location_ids:
+                    - 2 # Home (Content structure)
+                    - 5 # Users
+                    - 43 # Media
+                    - 48 # Setup
+                    - 55 # Forms
+                    - 56 # Site skeletons
+                    - 60 # Components
+                    - 67 # Dashboards
+            subtree_paths:
+                content: /1/2/
+                media: /1/43/
+            page_builder:
+                siteaccess_list: [ site ]
+            assets:
+                icon_sets:
+                    streamlineicons: /bundles/ibexaicons/img/all-icons.svg
+                default_icon_set: streamlineicons
+            content_type:
+                about:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#about'
+                article:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#article'
+                blog:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#blog'
+                blog_post:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#blog_post'
+                editor:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#author'
+                folder:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#folder'
+                form:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#form'
+                place:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#place'
+                product:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#product'
+                field:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#field'
+                user:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#user'
+                user_group:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#user_group'
+                file:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#file'
+                gallery:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#gallery'
+                image:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#image'
+                video:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#video'
+                landing_page:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#landing_page'
+                default-config:
+                    thumbnail: '/bundles/ibexaicons/img/all-icons.svg#file'
+            user_content_type_identifier: ['user', 'customer', 'member', 'editor']
+            user_profile:
+                enabled: true
+                content_types: ['editor']
+                field_groups: ['about', 'contact']
+            default_page: 'dashboard'
+            content:
+                default_ttl: 0
+            admin_ui_forms:
+                content_edit:
+                    form_templates:
+                        - { template: '@ibexadesign/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig', priority: -10 } }

code_samples/ai_actions/config/services.yaml

@@ -0,0 +1,79 @@
+# This file is the entry point to configure your own services.
+# Files in the packages/ subdirectory configure your dependencies.
+
+# Put parameters here that don't need to change on each machine where the app is deployed
+# https://symfony.com/doc/current/best_practices.html#use-parameters-for-application-configuration
+parameters:
+
+services:
+    # default configuration for services in *this* file
+    _defaults:
+        autowire: true      # Automatically injects dependencies in your services.
+        autoconfigure: true # Automatically registers your services as commands, event subscribers, etc.
+
+    # makes classes in src/ available to be used as services
+    # this creates a service per class whose id is the fully-qualified class name
+    App\:
+        resource: '../src/'
+        exclude:
+            - '../src/DependencyInjection/'
+            - '../src/Entity/'
+            - '../src/Kernel.php'
+
+    # add more service definitions when explicit configuration is needed
+    # please note that last definitions always *replace* previous ones
+
+    App\Command\AddMissingAltTextCommand:
+        arguments:
+            $projectDir: '%kernel.project_dir%'
+
+    App\AI\Handler\LLaVATextToTextActionHandler:
+        tags:
+            - { name: ibexa.ai.action.handler, priority: 0 }
+            - { name: ibexa.ai.action.handler.text_to_text, priority: 0 }
+
+    app.connector_ai.action_configuration.handler.llava_text_to_text.form_mapper.options:
+        class: Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\ActionHandlerOptionsFormMapper
+        arguments:
+            $formType: 'App\Form\Type\TextToTextOptionsType'
+        tags:
+            - name: ibexa.connector_ai.action_configuration.form_mapper.options
+              type: !php/const \App\AI\Handler\LLaVaTextToTextActionHandler::IDENTIFIER
+
+    App\AI\ActionType\TranscribeAudioActionType:
+        arguments:
+            $actionHandlers: !tagged_iterator
+                tag: app.connector_ai.action.handler.audio_to_text
+                default_index_method: getIdentifier
+                index_by: key
+        tags:
+            - { name: ibexa.ai.action.type, identifier: !php/const \App\AI\ActionType\TranscribeAudioActionType::IDENTIFIER }
+
+    app.connector_ai.action_configuration.handler.transcribe_audio.form_mapper.options:
+        class: Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\ActionTypeOptionsFormMapper
+        arguments:
+            $formType: 'App\Form\Type\TranscribeAudioOptionsType'
+        tags:
+            - name: ibexa.connector_ai.action_configuration.form_mapper.action_type_options
+              type: !php/const \App\AI\ActionType\TranscribeAudioActionType::IDENTIFIER
+
+    App\AI\Handler\WhisperAudioToTextActionHandler:
+        tags:
+            - { name: ibexa.ai.action.handler, priority: 0 }
+            - { name: app.connector_ai.action.handler.audio_to_text, priority: 0 }
+
+#REST services
+    App\AI\REST\Input\Parser\TranscribeAudio:
+        parent: Ibexa\ConnectorAi\REST\Input\Parser\Action
+        autowire: true
+        tags:
+            - { name: ibexa.rest.input.parser, mediaType: application/vnd.ibexa.api.ai.TranscribeAudio }
+
+    App\AI\REST\Output\Resolver\AudioTextResolver:
+        tags:
+            - { name: ibexa.ai.action.mime_type, key: application/vnd.ibexa.api.ai.AudioText }
+
+    App\AI\REST\Output\ValueObjectVisitor\AudioText:
+        parent: Ibexa\Contracts\Rest\Output\ValueObjectVisitor
+        tags:
+            - { name: ibexa.rest.output.value_object.visitor, type: App\AI\REST\Value\AudioText }

code_samples/ai_actions/src/AI/Action/TranscribeAudioAction.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace App\AI\Action;
+
+use App\AI\DataType\Audio;
+use Ibexa\Contracts\ConnectorAi\Action\Action;
+
+final class TranscribeAudioAction extends Action
+{
+    private Audio $audio;
+
+    public function __construct(Audio $audio)
+    {
+        $this->audio = $audio;
+    }
+
+    public function getParameters(): array
+    {
+        return [];
+    }
+
+    public function getInput(): Audio
+    {
+        return $this->audio;
+    }
+
+    public function getActionTypeIdentifier(): string
+    {
+        return 'transcribe_audio';
+    }
+}

code_samples/ai_actions/src/AI/ActionType/TranscribeAudioActionType.php

@@ -0,0 +1,67 @@
+<?php declare(strict_types=1);
+
+namespace App\AI\ActionType;
+
+use App\AI\Action\TranscribeAudioAction;
+use App\AI\DataType\Audio;
+use Ibexa\Contracts\ConnectorAi\Action\DataType\Text;
+use Ibexa\Contracts\ConnectorAi\ActionInterface;
+use Ibexa\Contracts\ConnectorAi\ActionType\ActionTypeInterface;
+use Ibexa\Contracts\ConnectorAi\DataType;
+use Ibexa\Contracts\Core\Exception\InvalidArgumentException;
+
+final class TranscribeAudioActionType implements ActionTypeInterface
+{
+    public const IDENTIFIER = 'transcribe_audio';
+
+    /** @var iterable<\Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface> */
+    private iterable $actionHandlers;
+
+    /** @param iterable<\Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface> $actionHandlers*/
+    public function __construct(iterable $actionHandlers)
+    {
+        $this->actionHandlers = $actionHandlers;
+    }
+
+    public function getIdentifier(): string
+    {
+        return self::IDENTIFIER;
+    }
+
+    public function getName(): string
+    {
+        return 'Transcribe audio';
+    }
+
+    public function getInputIdentifier(): string
+    {
+        return Audio::getIdentifier();
+    }
+
+    public function getOutputIdentifier(): string
+    {
+        return Text::getIdentifier();
+    }
+
+    public function getOptions(): array
+    {
+        return [];
+    }
+
+    public function createAction(DataType $input, array $parameters = []): ActionInterface
+    {
+        if (!$input instanceof Audio) {
+            throw new InvalidArgumentException(
+                'audio',
+                'expected \App\AI\DataType\Audio type, ' . get_debug_type($input) . ' given.'
+            );
+        }
+
+        return new TranscribeAudioAction($input);
+    }
+
+    public function getActionHandlers(): iterable
+    {
+        return $this->actionHandlers;
+    }
+}

code_samples/ai_actions/src/AI/DataType/Audio.php

@@ -0,0 +1,37 @@
+<?php declare(strict_types=1);
+
+namespace App\AI\DataType;
+
+use Ibexa\Contracts\ConnectorAi\DataType;
+
+/**
+ * @implements DataType<string>
+ */
+final class Audio implements DataType
+{
+    /** @var non-empty-array<string> */
+    private array $base64;
+
+    /**
+     * @param non-empty-array<string> $base64
+     */
+    public function __construct(array $base64)
+    {
+        $this->base64 = $base64;
+    }
+
+    public function getBase64(): string
+    {
+        return reset($this->base64);
+    }
+
+    public function getList(): array
+    {
+        return $this->base64;
+    }
+
+    public static function getIdentifier(): string
+    {
+        return 'audio';
+    }
+}