[terminal] WIP: Good Image Protocol (PoC)

Author: christianparpart

TODO.md

@@ -93,6 +93,10 @@
 - [ ] contour: provide `--mono` (or alike) CLI flag to "just" provide a QOpenGLWindow for best performance,
       lacking UI features as compromise.
 
+### Good Image Protocol
+
+- [ ] Make sure Screen::Image does not need to know about the underlying image format. (only the frontend needs to know about the actual format in use, so it can *render* the pixmaps)
+
 ### Usability Improvements
 
 - ? Images: copy action should uxe U+FFFC (object replacement) on grid cells that contain an image for text-based clipboard action

src/contour/ContourApp.cpp

@@ -19,9 +19,12 @@
 
 #include <terminal/Capabilities.h>
 #include <terminal/Parser.h>
+#include <terminal/Image.h>
 
 #include <crispy/StackTrace.h>
+#include <crispy/base64.h>
 #include <crispy/debuglog.h>
+#include <crispy/stdfs.h>
 #include <crispy/utils.h>
 
 #include <iostream>
@@ -35,16 +38,21 @@
 #include <sys/ioctl.h>
 #endif
 
+#define GOOD_IMAGE_PROTOCOL // TODO use cmake here instead
+
 using std::bind;
 using std::cerr;
 using std::cout;
+using std::ifstream;
 using std::make_unique;
 using std::ofstream;
 using std::string;
 using std::string_view;
 using std::unique_ptr;
+using std::vector;
 
 using namespace std::string_literals;
+using namespace std::string_view_literals;
 
 namespace CLI = crispy::cli;
 
@@ -202,6 +210,76 @@ namespace // {{{ helper
 #endif
 } // }}}
 
+#if defined(GOOD_IMAGE_PROTOCOL) // {{{
+terminal::ImageAlignment parseImageAlignment(string_view _text)
+{
+    (void) _text;
+    return terminal::ImageAlignment::TopStart; // TODO
+}
+
+terminal::ImageResize parseImageResize(string_view _text)
+{
+    (void) _text;
+    return terminal::ImageResize::NoResize; // TODO
+}
+
+terminal::Coordinate parsePosition(string_view _text)
+{
+    (void) _text;
+    return {}; // TODO
+}
+
+// TODO: chunkedFileReader(path) to return iterator over spans of data chunks.
+std::vector<uint8_t> readFile(FileSystem::path const& _path)
+{
+    auto ifs = ifstream(_path.string());
+    if (!ifs.good())
+        return {};
+
+    auto const size = FileSystem::file_size(_path);
+    auto text = vector<uint8_t>();
+    text.resize(size);
+    ifs.read((char*) &text[0], size);
+    return text;
+}
+
+void displayImage(terminal::ImageResize _resizePolicy,
+                  terminal::ImageAlignment _alignmentPolicy,
+                  crispy::Size _screenSize,
+                  string_view _fileName)
+{
+    auto constexpr ST = "\033\\"sv;
+
+    cout << fmt::format("{}f={},c={},l={},a={},z={};",
+        "\033Ps"sv,             // GIONESHOT
+        '0',                    // image format: 0 = auto detect
+        _screenSize.width,
+        _screenSize.height,
+        int(_alignmentPolicy),
+        int(_resizePolicy)
+    );
+
+#if 1
+    auto const data = readFile(_fileName);// TODO: incremental buffered read
+    auto encoderState = crispy::base64::EncoderState{};
+
+    vector<char> buf;
+    auto const writer = [&](string_view _data) { for (auto ch: _data) buf.push_back(ch); };
+    auto const flush = [&]() { cout.write(buf.data(), buf.size()); buf.clear(); };
+
+    for (uint8_t const byte: data)
+    {
+        crispy::base64::encode(byte, encoderState, writer);
+        if (buf.size() >= 4096)
+            flush();
+    }
+    flush();
+#endif
+
+    cout << ST;
+}
+#endif // }}}
+
 ContourApp::ContourApp() :
     App("contour", "Contour Terminal Emulator", CONTOUR_VERSION_STRING)
 {
@@ -301,6 +379,28 @@ int ContourApp::profileAction()
     return EXIT_SUCCESS;
 }
 
+#if defined(GOOD_IMAGE_PROTOCOL)
+crispy::Size parseSize(string_view _text)
+{
+    (void) _text;
+    return crispy::Size{};//TODO
+}
+
+int ContourApp::imageAction()
+{
+    auto const resizePolicy = parseImageResize(parameters().get<string>("contour.image.resize"));
+    auto const alignmentPolicy = parseImageAlignment(parameters().get<string>("contour.image.align"));
+    auto const size = parseSize(parameters().get<string>("contour.image.size"));
+    auto const fileName = parameters().verbatim.front();
+    // TODO: how do we wanna handle more than one verbatim arg (image)?
+    // => report error and EXIT_FAILURE as only one verbatim arg is allowed.
+    // FIXME: What if parameter `size` is given as `_size` instead, it should cause an
+    //        invalid-argument error above already!
+    displayImage(resizePolicy, alignmentPolicy, size, fileName);
+    return EXIT_SUCCESS;
+}
+#endif
+
 crispy::cli::Command ContourApp::parameterDefinition() const
 {
     return CLI::Command{
@@ -365,6 +465,37 @@ crispy::cli::Command ContourApp::parameterDefinition() const
                     }
                 }
             },
+#if defined(GOOD_IMAGE_PROTOCOL)
+            CLI::Command{
+                "image",
+                "Sends an image to the terminal emulator for display.",
+                CLI::OptionList{
+                    CLI::Option{"resize", CLI::Value{"fit"s},
+                        "Sets the image resize policy.\n"
+                        "Policies available are:\n"
+                        " - no (no resize),\n"
+                        " - fit (resize to fit),\n"
+                        " - fill (resize to fill),\n"
+                        " - stretch (stretch to fill)."
+                    },
+                    CLI::Option{"align", CLI::Value{"center"s},
+                        "Sets the image alignment policy.\n"
+                        "Possible policies are: TopLeft, TopCenter, TopRight, MiddleLeft, MiddleCenter, MiddleRight, BottomLeft, BottomCenter, BottomRight."
+                    },
+                    CLI::Option{"size", CLI::Value{""s},
+                        "Sets the amount of columns and rows to place the image onto. "
+                        "The top-left of the this area is the current cursor position, "
+                        "and it will be scrolled automatically if not enough rows are present."
+                    }
+                },
+                CLI::CommandList{},
+                CLI::CommandSelect::Explicit,
+                CLI::Verbatim{
+                    "IMAGE_FILE",
+                    "Path to image to be displayed. Image formats supported are at least PNG, JPG."
+                }
+            },
+#endif
             CLI::Command{
                 "capture",
                 "Captures the screen buffer of the currently running terminal.",

src/contour/ContourApp.h

@@ -35,6 +35,7 @@ class ContourApp : public crispy::App
     int terminfoAction();
     int configAction();
     int integrationAction();
+    int imageAction();
 };
 
 }

src/contour/opengl/TerminalWidget.cpp

@@ -1117,6 +1117,50 @@ void TerminalWidget::updateMinimumSize()
     };
     setMinimumSize(minSize.width.as<int>(), minSize.height.as<int>());
 }
+
+optional<terminal::Image> TerminalWidget::decodeImage(crispy::span<uint8_t> _imageData)
+{
+    QImage image;
+    image.loadFromData(_imageData.begin(), _imageData.size());
+
+    qDebug() << "decodeImage()" << image.format();
+    if (image.hasAlphaChannel() && image.format() != QImage::Format_ARGB32)
+        image = image.convertToFormat(QImage::Format_ARGB32);
+    else
+        image = image.convertToFormat(QImage::Format_RGB888);
+    qDebug() << "|> decodeImage()" << image.format()
+        << image.sizeInBytes()
+        << image.size()
+        ;
+
+    static terminal::Image::Id nextImageId = 0;
+
+    terminal::Image::Data pixels;
+    auto* p = &pixels[0];
+    pixels.resize(image.bytesPerLine() * image.height());
+    for (int i = 0; i < image.height(); ++i)
+    {
+        memcpy(p, image.constScanLine(i), image.bytesPerLine());
+        p += image.bytesPerLine();
+    }
+
+    terminal::ImageFormat format = terminal::ImageFormat::RGBA;
+    switch (image.format())
+    {
+        case QImage::Format_RGBA8888:
+            format = terminal::ImageFormat::RGBA;
+            break;
+        case QImage::Format_RGB888:
+            format = terminal::ImageFormat::RGB;
+            break;
+        default:
+            return nullopt;
+    }
+    auto const size = ImageSize{Width(image.width()), Height(image.height())};
+
+    auto img = terminal::Image(nextImageId++, format, std::move(pixels), size);
+    return {std::move(img)};
+}
 // }}}
 
 } // namespace contour

src/contour/opengl/TerminalWidget.h

@@ -20,6 +20,7 @@
 #include <contour/helper.h>
 
 #include <terminal/Color.h>
+#include <terminal/Image.h>
 #include <terminal/Metrics.h>
 #include <terminal/primitives.h>
 #include <terminal_renderer/Renderer.h>
@@ -66,6 +67,11 @@ class TerminalWidget :
     static QSurfaceFormat surfaceFormat();
     QSize minimumSizeHint() const override;
     QSize sizeHint() const override;
+
+    std::optional<terminal::Image> decodeImage(crispy::span<uint8_t> _imageData);
+
+    int pointsToPixels(text::font_size _fontSize) const noexcept;
+
     void initializeGL() override;
     void resizeGL(int _width, int _height) override;
     void paintGL() override;

src/terminal/CMakeLists.txt

@@ -17,6 +17,7 @@ option(LIBTERMINAL_PASSIVE_RENDER_BUFFER_UPDATE "Updates the render buffer withi
 # Compile-time terminal features
 option(LIBTERMINAL_IMAGES "Enables image support [default: ON]" ON)
 option(LIBTERMINAL_HYPERLINKS "Enables hyperlink support [default: ON]" ON)
+option(GOOD_IMAGE_PROTOCOL "Enables building with Good Image Protocol support [default: OFF]" OFF)
 
 if(MSVC)
     add_definitions(-DNOMINMAX)
@@ -33,6 +34,7 @@ set(terminal_HEADERS
     InputBinding.h
     InputGenerator.h
     MatchModes.h
+    MessageParser.h
     Parser.h
     Process.h
     pty/Pty.h
@@ -63,6 +65,7 @@ set(terminal_SOURCES
     InputBinding.cpp
     InputGenerator.cpp
     MatchModes.cpp
+    MessageParser.cpp
     Parser.cpp
     Process.cpp
     RenderBuffer.cpp
@@ -104,6 +107,9 @@ endif()
 if(LIBTERMINAL_LOG_TRACE)
     target_compile_definitions(terminal PRIVATE LIBTERMINAL_LOG_TRACE=1)
 endif()
+if(GOOD_IMAGE_PROTOCOL)
+    target_compile_definitions(terminal PUBLIC GOOD_IMAGE_PROTOCOL=1)
+endif()
 
 if(LIBTERMINAL_IMAGES)
     target_compile_definitions(terminal PUBLIC LIBTERMINAL_IMAGES=1)
@@ -126,6 +132,7 @@ if(LIBTERMINAL_TESTING)
 		Selector_test.cpp
         Functions_test.cpp
         Grid_test.cpp
+        MessageParser_test.cpp
         Parser_test.cpp
         Screen_test.cpp
         Terminal_test.cpp

src/terminal/Functions.h

@@ -366,6 +366,15 @@ constexpr inline auto RCOLORHIGHLIGHTBG = detail::OSC(117, "RCOLORHIGHLIGHTBG",
 constexpr inline auto NOTIFY        = detail::OSC(777, "NOTIFY", "Send Notification.");
 constexpr inline auto DUMPSTATE     = detail::OSC(888, "DUMPSTATE", "Dumps internal state to debug stream.");
 
+// DCS: Good Image Protocol
+#if defined(GOOD_IMAGE_PROTOCOL)
+// TODO: use OSC instead of DCS?
+constexpr inline auto GIUPLOAD    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'u', VTType::VT525, "GIUPLOAD", "Uploads an image.");
+constexpr inline auto GIRENDER    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'r', VTType::VT525, "GIRENDER", "Renders an image.");
+constexpr inline auto GIDELETE    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'd', VTType::VT525, "GIDELETE", "Deletes an image.");
+constexpr inline auto GIONESHOT   = detail::DCS(std::nullopt, 0, 0, std::nullopt, 's', VTType::VT525, "GIONESHOT", "Uploads and renders an unnamed image.");
+#endif
+
 inline auto const& functions() noexcept
 {
     static auto const funcs = []() constexpr { // {{{
@@ -465,6 +474,12 @@ inline auto const& functions() noexcept
             XTVERSION,
 
             // DCS
+#if defined(GOOD_IMAGE_PROTOCOL)
+            GIUPLOAD,
+            GIRENDER,
+            GIDELETE,
+            GIONESHOT,
+#endif
             STP,
             DECRQSS,
             DECSIXEL,

src/terminal/Image.h

@@ -75,7 +75,7 @@ class Image {
     Id id_;
     ImageFormat format_;
     Data data_;
-    crispy::Size size_;
+    ImageSize size_;
 };
 
 /// Image resize hints are used to properly fit/fill the area to place the image onto.