Skip to content

Finally, correct UTF-8 and charset handling: #1759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
xsawyerx wants to merge 8 commits into from
Closed

Finally, correct UTF-8 and charset handling: #1759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5c0bcba
Finally, correct UTF-8 and charset handling:
xsawyerx Jan 21, 2026
1b85573
Fix warnings in test:
xsawyerx Jan 22, 2026
38fd7cf
Get some verbosity
xsawyerx Jan 22, 2026
9d122f3
Fix utf8 guarding with Encode when Unicode::UTF8 missing:
xsawyerx Jan 22, 2026
3933cf4
No need for explicit copy
xsawyerx Jan 22, 2026
6c7513f
Ensure UTF‑8 encoding at response edge, warn once via logger:
xsawyerx Jan 22, 2026
ee55cfc
Adding test from GH #1143
xsawyerx Jan 26, 2026
1fc4149
Default charset to UTF‑8:
xsawyerx Jan 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ jobs:
run: |
perl -v
cpanm --notest --force Module::Pluggable
cpanm Dancer2-*.tar.gz
cpanm -v Dancer2-*.tar.gz
perl -MDancer2 -e 'print "$Dancer2::VERSION\n"'


Expand Down Expand Up @@ -141,7 +141,7 @@ jobs:
run: |
perl -v
cpanm --notest --force Module::Pluggable
cpanm Dancer2-*.tar.gz
cpanm -v Dancer2-*.tar.gz
perl -MDancer2 -e 'print "$Dancer2::VERSION\n"'

- name: Testing selected Plugins
Expand Down
39 changes: 31 additions & 8 deletions lib/Dancer2/Core/App.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,9 @@ has '+local_triggers' => (

# set the engine with the new value from the builder
$self->$setter_method($engine_instance);
if ( $engine eq 'serializer' && $self->has_response ) {
$self->response->serializer($engine_instance);
}

return $engine_instance;
};
Expand Down Expand Up @@ -287,10 +290,11 @@ sub _build_template_engine {
$self->_get_config_for_engine( template => $value, $config );

my $engine_attrs = {
config => $engine_options,
layout => $config->{layout},
config => $engine_options,
layout => $config->{layout},
layout_dir => ( $config->{layout_dir} || 'layouts' ),
views => $config->{views},
views => $config->{views},
charset => $config->{charset},
};

Scalar::Util::weaken( my $weak_self = $self );
Expand All @@ -317,6 +321,7 @@ sub _build_serializer_engine {

my $engine_options =
$self->_get_config_for_engine( serializer => $value, $config );
$engine_options->{strict_utf8} //= $config->{strict_utf8};

Scalar::Util::weaken( my $weak_self = $self );

Expand Down Expand Up @@ -463,6 +468,12 @@ sub _build_response {
return Dancer2::Core::Response->new(
mime_type => $self->mime_type,
server_tokens => !$self->config->{'no_server_tokens'},
charset => $self->config->{charset},
strict_utf8 => $self->config->{strict_utf8},
do {
Scalar::Util::weaken( my $weak_self = $self );
log_cb => sub { $weak_self && $weak_self->log(@_) };
},
$self->has_serializer_engine
? ( serializer => $self->serializer_engine )
: (),
Expand Down Expand Up @@ -733,7 +744,8 @@ sub _build_default_config {
my $public = $ENV{DANCER_PUBLIC} || path( $self->location, 'public' );
return {
content_type => ( $ENV{DANCER_CONTENT_TYPE} || 'text/html' ),
charset => ( $ENV{DANCER_CHARSET} || '' ),
charset => ( $ENV{DANCER_CHARSET} || 'UTF-8' ),
strict_utf8 => ( $ENV{DANCER_STRICT_UTF8} || 0 ),
logger => ( $ENV{DANCER_LOGGER} || 'console' ),
views => ( $ENV{DANCER_VIEWS}
|| path( $self->location, 'views' ) ),
Expand Down Expand Up @@ -1008,8 +1020,11 @@ sub send_as {
carp sprintf( "Please use %s as the type for 'send_as', not %s", lc($type), $type );
}

$options->{charset} = $self->config->{charset} || 'UTF-8';
my $content = Encode::encode( $options->{charset}, $data );
$options->{charset} //= $self->config->{charset};
my $content = $data;
if ( defined $options->{charset} && length $options->{charset} ) {
$content = Encode::encode( $options->{charset}, $data );
}
$options->{content_type} ||= join '/', 'text', lc $type;
# Explicit return needed here, as if we are currently rendering a
# template then with_return will not longjump
Expand All @@ -1031,6 +1046,7 @@ sub send_as {
# load any serializer engine config
my $engine_options =
$self->_get_config_for_engine( serializer => $type, $self->config ) || {};
$engine_options->{strict_utf8} //= $self->config->{strict_utf8};

Scalar::Util::weaken( my $weak_self = $self );
my $serializer = $self->_factory->create(
Expand All @@ -1051,6 +1067,7 @@ sub send_error {
my $err = Dancer2::Core::Error->new(
message => $message,
app => $self,
charset => $self->config->{charset},
( status => $status )x!! $status,

$self->has_serializer_engine
Expand Down Expand Up @@ -1119,7 +1136,7 @@ sub send_file {
binmode $fh;
$content_type = $self->mime_type->for_file($file_path) || 'text/plain';
if ( $content_type =~ m!^text/! ) {
$charset = $self->config->{charset} || "utf-8";
$charset = $self->config->{charset};
}
}

Expand Down Expand Up @@ -1539,6 +1556,7 @@ sub dispatch {
app => $app,
message => $err,
status => 400, # 400 Bad request (dont send again), rather than 500
charset => $self->config->{charset},
)->throw;
}

Expand Down Expand Up @@ -1682,6 +1700,7 @@ sub build_request {
env => $env,
is_behind_proxy => $self->settings->{'behind_proxy'} || 0,
uri_for_route => sub { shift; $weak_self->uri_for_route(@_) },
strict_utf8 => $self->config->{strict_utf8},

$self->has_serializer_engine
? ( serializer => $self->serializer_engine )
Expand Down Expand Up @@ -1733,6 +1752,8 @@ sub _prep_response {
and my $ct = $config->{content_type} ) {
$response->default_content_type($ct);
}
exists $config->{charset}
Comment thread
xsawyerx marked this conversation as resolved.
and $response->charset( $config->{charset} );

# if we were passed any content, set it in the response
defined $content && $response->content($content);
Expand All @@ -1752,6 +1773,7 @@ sub response_internal_error {
app => $self,
status => 500,
exception => $error,
charset => $self->config->{charset},
)->throw;
}

Expand All @@ -1764,9 +1786,10 @@ sub response_not_found {
local $Dancer2::Core::Route::RESPONSE = $self->response;

my $response = Dancer2::Core::Error->new(
app => $self,
app => $self,
status => 404,
message => $request->path,
charset => $self->config->{charset},
)->throw;

$self->cleanup;
Expand Down
3 changes: 2 additions & 1 deletion lib/Dancer2/Core/Error.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ sub throw {

$self->response->status( $self->status );
$self->response->content_type( $self->content_type );
$self->response->charset( $self->charset ) if defined $self->charset;
$self->response->content($message);

$self->has_app &&
Expand Down Expand Up @@ -392,7 +393,7 @@ sub backtrace {
return $html unless $file and $line;

# file and line are located, let's read the source Luke!
my $fh = eval { open_file('<', $file) } or return $html;
my $fh = eval { open_file( '<', $file, $self->charset ) } or return $html;
my @lines = <$fh>;
close $fh;

Expand Down
84 changes: 68 additions & 16 deletions lib/Dancer2/Core/Request.pm
View file
Open in desktop
Comment thread
xsawyerx marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use warnings;
use parent 'Plack::Request';

use Carp;
use Encode;
use Encode qw(decode FB_CROAK LEAVE_SRC);
use URI;
use URI::Escape;
use Safe::Isa;
Expand Down Expand Up @@ -40,10 +40,14 @@ eval {
require Unicode::UTF8;
no warnings qw<redefine once>;
*__decode = sub { Unicode::UTF8::decode_utf8($_[0]) };
*__valid = sub { Unicode::UTF8::valid_utf8($_[0]) };
1;
} or do {
no warnings qw<redefine once>;
*__decode = sub { decode( 'UTF-8', $_[0] ) };
*__valid = sub {
eval { decode( 'UTF-8', $_[0], FB_CROAK | LEAVE_SRC ); 1 };
};
};

# check presence of XS module to speedup request
Expand Down Expand Up @@ -81,6 +85,7 @@ sub new {

$opts{'body_params'}
and $self->{'_body_params'} = $opts{'body_params'};
$self->{'_strict_utf8'} = !!$opts{'strict_utf8'};

# Deserialize/parse body for HMV
$self->data;
Expand Down Expand Up @@ -126,14 +131,14 @@ sub _query_params { $_[0]->{'_query_params'} }

sub _set_query_params {
my ( $self, $params ) = @_;
$self->{_query_params} = _decode( $params );
$self->{_query_params} = $self->_decode( $params, 'query parameters' );
}

sub _route_params { $_[0]->{'_route_params'} ||= {} }

sub _set_route_params {
my ( $self, $params ) = @_;
$self->{_route_params} = _decode( $params );
$self->{_route_params} = $self->_decode( $params, 'route parameters' );
$self->_build_params();
}

Expand Down Expand Up @@ -294,6 +299,12 @@ sub uri_base {
return $canon;
}

sub path {
my $self = shift;
my $path = $self->env->{PATH_INFO} || '/';
return $self->_decode_bytes( $path, 'PATH_INFO' );
}

sub dispatch_path {
Carp::croak q{DEPRECATED: request->dispatch_path. Please use request->path instead};
}
Expand Down Expand Up @@ -353,9 +364,9 @@ sub query_parameters {
my $self = shift;
$self->{'query_parameters'} ||= do {
if ($XS_PARSE_QUERY_STRING) {
my $query = _decode(CGI::Deurl::XS::parse_query_string(
my $query = $self->_decode(CGI::Deurl::XS::parse_query_string(
$self->env->{'QUERY_STRING'}
));
), 'query parameters');

Hash::MultiValue->new(
map {;
Expand All @@ -367,7 +378,7 @@ sub query_parameters {
);
} else {
# defer to Plack::Request
_decode($self->SUPER::query_parameters);
$self->_decode($self->SUPER::query_parameters, 'query parameters');
}
};
}
Expand All @@ -380,13 +391,18 @@ sub _set_route_parameters {
# remove reserved splat parameter name
# you should access splat parameters using splat() keyword
delete @{$params}{qw<splat captures>};
$self->{'route_parameters'} = Hash::MultiValue->from_mixed( %{_decode($params)} );
$self->{'route_parameters'} = Hash::MultiValue->from_mixed(
%{ $self->_decode( $params, 'route parameters' ) }
);
}

sub body_parameters {
my $self = shift;
# defer to (the overridden) Plack::Request->body_parameters
$self->{'body_parameters'} ||= _decode($self->SUPER::body_parameters());
$self->{'body_parameters'} ||= $self->_decode(
$self->SUPER::body_parameters(),
'body parameters',
);
}

sub parameters {
Expand Down Expand Up @@ -415,25 +431,59 @@ sub splat { @{ shift->params->{splat} || [] } }
sub param { shift->params->{ $_[0] } }

sub _decode {
my ($h) = @_;
my ( $self, $h, $context ) = @_;
return if not defined $h;

if ( !is_ref($h) && !utf8::is_utf8($h) ) {
return __decode($h);
return $self->_decode_bytes( $h, $context );
}
elsif ( ref($h) eq 'Hash::MultiValue' ) {
return Hash::MultiValue->from_mixed(_decode($h->as_hashref_mixed));
return Hash::MultiValue->from_mixed(
$self->_decode( $h->as_hashref_mixed, $context )
);
}
elsif ( is_hashref($h) ) {
return { map {my $t = _decode($_); $t} (%$h) };
return { map scalar $self->_decode( $_, $context ), %{$h} };
}
elsif ( is_arrayref($h) ) {
return [ map _decode($_), @$h ];
return [ map $self->_decode( $_, $context ), @{$h} ];
}

return $h;
}

sub _decode_bytes {
my ( $self, $bytes, $context ) = @_;

# If PSGI already gave us characters, avoid re-decoding.
return $bytes if utf8::is_utf8($bytes);
return $bytes if $bytes !~ /[\x80-\xFF]/;

return __decode($bytes) if __valid($bytes);
return $self->_invalid_utf8( $bytes, $context );
}

sub _invalid_utf8 {
my ( $self, $bytes, $context ) = @_;
my $strict = $self->{_strict_utf8};
my $where = $context || 'input';
my $msg = "Invalid UTF-8 in $where";

$strict
and Carp::croak($msg);

if ( my $logger = $self->env->{'psgix.logger'} ) {
$logger->({
level => 'warning',
message => "$msg; leaving bytes unchanged",
});
} else {
Carp::carp("$msg; leaving bytes unchanged");
}

return $bytes;
}

sub is_ajax {
my $self = shift;

Expand Down Expand Up @@ -542,7 +592,7 @@ sub _build_uploads {
headers => {@{$_->{headers}->psgi_flatten_without_sort}},
tempname => $_->{tempname},
size => $_->{size},
filename => _decode( $_->{filename} ),
filename => $self->_decode( $_->{filename}, 'upload filename' ),
), $uploads->get_all($name);

$uploads{$name} = @uploads > 1 ? \@uploads : $uploads[0];
Expand Down Expand Up @@ -601,6 +651,7 @@ sub _shallow_clone {
# Clone and merge query params
my $new_params = $self->params;
$new_request->{_query_params} = { %{ $self->{_query_params} || {} } };
$new_request->{_strict_utf8} = $self->{_strict_utf8};
$new_request->{query_parameters} = $self->query_parameters->clone;
for my $key ( keys %{ $params || {} } ) {
my $value = $params->{$key};
Expand Down Expand Up @@ -961,8 +1012,9 @@ associated risks and alternatives.

=method path

The path requested by the client, normalized. This is effectively
C<path_info> or a single forward C</>.
The decoded path requested by the client, normalized. This is effectively
C<path_info> or a single forward C</>. Invalid UTF-8 is left as bytes in
lenient mode (with a warning), or throws an error in strict UTF-8 mode.

=method path_info

Expand Down
Loading
Loading