From a64eaf63b9e70e502d64f489a217fd6c4ede6c31 Mon Sep 17 00:00:00 2001
From: Leon Timmermans <fawaka@gmail.com>
Date: Thu, 28 Dec 2017 19:23:03 +0100
Subject: [PATCH] PerlIO::encoding has a $fallback variable that allows one to
 set the behavior on a encoding/decoding error, for example to make it throw
 an exception on error. What is not documented (actually the example in the
 documentation is even missing this) is that PerlIO::encoding needs the
 (equally undocumented) Encode::STOP_AT_PARTIAL flag to be set, otherwise a
 multi-byte character spanning buffer boundaries will be interpreted as two
 invalid byte sequences. I could have fixed the documentation, but instead I
 fixed the code to always pass this flag to Encode, simplifying the use and
 making the current documentation correct again.

---
 ext/PerlIO-encoding/encoding.pm |  5 ++---
 ext/PerlIO-encoding/encoding.xs | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/ext/PerlIO-encoding/encoding.pm b/ext/PerlIO-encoding/encoding.pm
index 08d2df4713..c3b36df2ee 100644
--- a/ext/PerlIO-encoding/encoding.pm
+++ b/ext/PerlIO-encoding/encoding.pm
@@ -1,7 +1,7 @@
 package PerlIO::encoding;
 
 use strict;
-our $VERSION = '0.25';
+our $VERSION = '0.26';
 our $DEBUG = 0;
 $DEBUG and warn __PACKAGE__, " called by ", join(", ", caller), "\n";
 
@@ -13,8 +13,7 @@ $DEBUG and warn __PACKAGE__, " called by ", join(", ", caller), "\n";
 require XSLoader;
 XSLoader::load();
 
-our $fallback =
-    Encode::PERLQQ()|Encode::WARN_ON_ERR()|Encode::STOP_AT_PARTIAL();
+our $fallback = Encode::PERLQQ()|Encode::WARN_ON_ERR();
 
 1;
 __END__
diff --git a/ext/PerlIO-encoding/encoding.xs b/ext/PerlIO-encoding/encoding.xs
index bb4754f3d9..3445082a7c 100644
--- a/ext/PerlIO-encoding/encoding.xs
+++ b/ext/PerlIO-encoding/encoding.xs
@@ -5,6 +5,10 @@
 #define U8 U8
 
 #define OUR_DEFAULT_FB	"Encode::PERLQQ"
+#define OUR_STOP_AT_PARTIAL "Encode::STOP_AT_PARTIAL"
+
+/* This will be set during BOOT */
+static unsigned int encode_stop_at_partial = 0;
 
 #if defined(USE_PERLIO)
 
@@ -164,6 +168,7 @@ PerlIOEncode_pushed(pTHX_ PerlIO * f, const char *mode, SV * arg, PerlIO_funcs *
     }
 
     e->chk = newSVsv(get_sv("PerlIO::encoding::fallback", 0));
+    SvIV_set(e->chk, SvIV(e->chk) | encode_stop_at_partial);
     e->inEncodeCall = 0;
 
     FREETMPS;
@@ -685,6 +690,16 @@ BOOT:
     }
     SPAGAIN;
     sv_setsv(chk, POPs);
+
+    PUSHMARK(sp);
+    PUTBACK;
+    if (call_pv(OUR_STOP_AT_PARTIAL, G_SCALAR) != 1) {
+	    /* should never happen */
+	    Perl_die(aTHX_ "%s did not return a value", OUR_STOP_AT_PARTIAL);
+    }
+    SPAGAIN;
+    encode_stop_at_partial = POPu;
+
     PUTBACK;
 #ifdef PERLIO_LAYERS
     PerlIO_define_layer(aTHX_ PERLIO_FUNCS_CAST(&PerlIO_encode));
-- 
2.15.1

From 35d99902af4832a40c4aa9d88895f98aa0b22755 Mon Sep 17 00:00:00 2001
From: Leon Timmermans <fawaka@gmail.com>
Date: Thu, 28 Dec 2017 19:23:03 +0100
Subject: [PATCH 1/2] Enforce STOP_AT_PARTIAL in $PerlIO::encoding::fallback

PerlIO::encoding has a $fallback variable that allows one to set the
behavior on a encoding/decoding error, for example to make it throw an
exception on error. What is not documented (actually the example in the
documentation is even missing this) is that PerlIO::encoding needs the
(equally undocumented) Encode::STOP_AT_PARTIAL flag to be set, otherwise
a multi-byte character spanning buffer boundaries will be interpreted as
two invalid byte sequences. I could have fixed the documentation, but
instead I fixed the code to always pass this flag to Encode, simplifying
the use and making the current documentation correct again.
---
 ext/PerlIO-encoding/encoding.pm |  5 ++---
 ext/PerlIO-encoding/encoding.xs | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/ext/PerlIO-encoding/encoding.pm b/ext/PerlIO-encoding/encoding.pm
index 08d2df4713..c3b36df2ee 100644
--- a/ext/PerlIO-encoding/encoding.pm
+++ b/ext/PerlIO-encoding/encoding.pm
@@ -1,7 +1,7 @@
 package PerlIO::encoding;
 
 use strict;
-our $VERSION = '0.25';
+our $VERSION = '0.26';
 our $DEBUG = 0;
 $DEBUG and warn __PACKAGE__, " called by ", join(", ", caller), "\n";
 
@@ -13,8 +13,7 @@ $DEBUG and warn __PACKAGE__, " called by ", join(", ", caller), "\n";
 require XSLoader;
 XSLoader::load();
 
-our $fallback =
-    Encode::PERLQQ()|Encode::WARN_ON_ERR()|Encode::STOP_AT_PARTIAL();
+our $fallback = Encode::PERLQQ()|Encode::WARN_ON_ERR();
 
 1;
 __END__
diff --git a/ext/PerlIO-encoding/encoding.xs b/ext/PerlIO-encoding/encoding.xs
index bb4754f3d9..39ac40f182 100644
--- a/ext/PerlIO-encoding/encoding.xs
+++ b/ext/PerlIO-encoding/encoding.xs
@@ -5,6 +5,10 @@
 #define U8 U8
 
 #define OUR_DEFAULT_FB	"Encode::PERLQQ"
+#define OUR_STOP_AT_PARTIAL "Encode::STOP_AT_PARTIAL"
+
+/* This will be set during BOOT */
+static unsigned int encode_stop_at_partial = 0;
 
 #if defined(USE_PERLIO)
 
@@ -164,6 +168,7 @@ PerlIOEncode_pushed(pTHX_ PerlIO * f, const char *mode, SV * arg, PerlIO_funcs *
     }
 
     e->chk = newSVsv(get_sv("PerlIO::encoding::fallback", 0));
+    SvUV_set(e->chk, SvUV(e->chk) | encode_stop_at_partial);
     e->inEncodeCall = 0;
 
     FREETMPS;
@@ -685,6 +690,16 @@ BOOT:
     }
     SPAGAIN;
     sv_setsv(chk, POPs);
+
+    PUSHMARK(sp);
+    PUTBACK;
+    if (call_pv(OUR_STOP_AT_PARTIAL, G_SCALAR) != 1) {
+	    /* should never happen */
+	    Perl_die(aTHX_ "%s did not return a value", OUR_STOP_AT_PARTIAL);
+    }
+    SPAGAIN;
+    encode_stop_at_partial = POPu;
+
     PUTBACK;
 #ifdef PERLIO_LAYERS
     PerlIO_define_layer(aTHX_ PERLIO_FUNCS_CAST(&PerlIO_encode));
-- 
2.15.0-291-g0d8980c

From f4e72dbfa4b0d957dd2dcfcf63b615ddd38ae786 Mon Sep 17 00:00:00 2001
From: Leon Timmermans <fawaka@gmail.com>
Date: Thu, 4 Jan 2018 19:56:03 +0100
Subject: [PATCH 2/2] Disallow coderef in $PerlIO::encoding::fallback

Encode allows one to pass a coderef instead of a set of flags to handle.
This however doesn't allow one to pass STOP_AT_PARTIAL, which means it
has always been buggy on buffer boundaries. With my new automatic
STOP_AT_PARTIAL passing this would result in an unpredictable value.
Instead we now disallow it in PerlIO::encoding.
---
 ext/PerlIO-encoding/encoding.xs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ext/PerlIO-encoding/encoding.xs b/ext/PerlIO-encoding/encoding.xs
index 39ac40f182..66728734d6 100644
--- a/ext/PerlIO-encoding/encoding.xs
+++ b/ext/PerlIO-encoding/encoding.xs
@@ -168,6 +168,8 @@ PerlIOEncode_pushed(pTHX_ PerlIO * f, const char *mode, SV * arg, PerlIO_funcs *
     }
 
     e->chk = newSVsv(get_sv("PerlIO::encoding::fallback", 0));
+    if (SvROK(e->chk))
+	Perl_croak(aTHX_ "PerlIO::encoding::fallback must be an integer");
     SvUV_set(e->chk, SvUV(e->chk) | encode_stop_at_partial);
     e->inEncodeCall = 0;
 
-- 
2.15.0-291-g0d8980c