formatting and utf-8

Author: dorianmariecom

lib/code/format.rb

@@ -434,7 +434,7 @@ def format_left_operation(operation, indent:)
 
       Array(operation[:others]).each do |other|
         right = format_nested_statement(other[:statement], indent: indent)
-        operator = other[:operator]
+        operator = format_operator(other[:operator])
 
         expression =
           if compact_operator?(operator)
@@ -532,7 +532,7 @@ def extract_string_concatenation_parts(statement)
     end
 
     def format_right_operation(operation, indent:)
-      operator = operation[:operator].to_s
+      operator = format_operator(operation[:operator])
       left =
         if %w[if unless].include?(operator)
           format_modifier_left(operation[:left], indent: indent)
@@ -595,6 +595,17 @@ def compact_operator?(operator)
       %w[. :: &. .. ...].include?(operator)
     end
 
+    def format_operator(operator)
+      case operator.to_s
+      when "||"
+        "or"
+      when "&&"
+        "and"
+      else
+        operator.to_s
+      end
+    end
+
     def format_ternary(ternary, indent:)
       left = format_nested_statement(ternary[:left], indent: indent)
       middle = format_nested_statement(ternary[:middle], indent: indent)

lib/code/object/string.rb

@@ -157,6 +157,10 @@ def code_size
 
       def code_strip
         String.new(raw.strip)
+      rescue ArgumentError, Encoding::CompatibilityError => e
+        raise unless e.message.include?("invalid byte sequence")
+
+        String.new(sanitized_utf8_raw.strip)
       end
 
       def code_split(value)
@@ -172,6 +176,15 @@ def code_split(value)
       def present?
         raw.present?
       end
+
+      private
+
+      def sanitized_utf8_raw
+        raw
+          .dup
+          .force_encoding(::Encoding::UTF_8)
+          .encode(::Encoding::UTF_8, invalid: :replace, undef: :replace)
+      end
     end
   end
 end

spec/code/format_spec.rb

@@ -13,6 +13,8 @@
       %w[100000 100_000],
       %w[1000000 1_000_000],
       %w[1.0000000001 1.000_000_000_1],
+      ["true || false", "true or false"],
+      ["true && false", "true and false"],
       ["{a:1}", "{ a: 1 }"],
       ["[1,2,3]", "[1, 2, 3]"],
       ["[1, 2, 3].select { |n| n.even? }", "[1, 2, 3].select { |n| n.even? }"],
@@ -54,7 +56,7 @@
       ],
       [
         "sections << Html.join([Html.p { Html.b { \"{index + 1}. {title}\" } }, Html.p { query } if query.presence, Html.p { Html.a(href: link || inline_url) { :source } } if (link || inline_url), Html.p { Html.a(href: inline_url) { Html.img(src: inline_url, alt: title) } }, Html.p { Html.a(href: attachment_url) { \"télécharger\" } }].compact)",
-        "sections << Html.join(\n  [\n    Html.p { Html.b { \"{index + 1}. {title}\" } },\n    Html.p { query } if query.presence,\n    Html.p {\n      Html.a(href: link || inline_url) { :source }\n    } if (link || inline_url),\n    Html.p {\n      Html.a(href: inline_url) { Html.img(src: inline_url, alt: title) }\n    },\n    Html.p {\n      Html.a(href: attachment_url) { \"télécharger\" }\n    }\n  ].compact\n)"
+        "sections << Html.join(\n  [\n    Html.p { Html.b { \"{index + 1}. {title}\" } },\n    Html.p { query } if query.presence,\n    Html.p {\n      Html.a(href: link or inline_url) { :source }\n    } if (link or inline_url),\n    Html.p {\n      Html.a(href: inline_url) { Html.img(src: inline_url, alt: title) }\n    },\n    Html.p {\n      Html.a(href: attachment_url) { \"télécharger\" }\n    }\n  ].compact\n)"
       ],
       [
         "safe = post.present? and !post[:over_18] and post[:post_hint] == :image and post[:url].to_string.strip.presence and (post[:url].to_string.strip.ends_with?(\".jpg\") or post[:url].to_string.strip.ends_with?(\".jpeg\") or post[:url].to_string.strip.ends_with?(\".png\") or post[:url].to_string.strip.include?(\"i.redd.it\"))",

spec/code/object/string_spec.rb

@@ -15,4 +15,12 @@
       expect(Code.evaluate(input).to_s).to eq(expected)
     end
   end
+
+  describe "#code_strip" do
+    it "replaces invalid utf-8 bytes instead of raising" do
+      string = described_class.new("\xC3 ".b.force_encoding(Encoding::UTF_8))
+
+      expect(string.code_strip.to_s).to eq("\uFFFD")
+    end
+  end
 end