| Current Path : /usr/opt/perl530/man/man1/ |
FreeBSD hs32.drive.ne.jp 9.1-RELEASE FreeBSD 9.1-RELEASE #1: Wed Jan 14 12:18:08 JST 2015 root@hs32.drive.ne.jp:/sys/amd64/compile/hs32 amd64 |
| Current File : //usr/opt/perl530/man/man1/perlunicook.1 |
.\" Automatically generated by Pod::Man 4.11 (Pod::Simple 3.35)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings. \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote. \*(C+ will
.\" give a nicer C++. Capital omega is used to do unbreakable dashes and
.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
. ds -- \(*W-
. ds PI pi
. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch
. ds L" ""
. ds R" ""
. ds C` ""
. ds C' ""
'br\}
.el\{\
. ds -- \|\(em\|
. ds PI \(*p
. ds L" ``
. ds R" ''
. ds C`
. ds C'
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el .ds Aq '
.\"
.\" If the F register is >0, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD. Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.\"
.\" Avoid warning from groff about undefined register 'F'.
.de IX
..
.nr rF 0
.if \n(.g .if rF .nr rF 1
.if (\n(rF:(\n(.g==0)) \{\
. if \nF \{\
. de IX
. tm Index:\\$1\t\\n%\t"\\$2"
..
. if !\nF==2 \{\
. nr % 0
. nr F 2
. \}
. \}
.\}
.rr rF
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear. Run. Save yourself. No user-serviceable parts.
. \" fudge factors for nroff and troff
.if n \{\
. ds #H 0
. ds #V .8m
. ds #F .3m
. ds #[ \f1
. ds #] \fP
.\}
.if t \{\
. ds #H ((1u-(\\\\n(.fu%2u))*.13m)
. ds #V .6m
. ds #F 0
. ds #[ \&
. ds #] \&
.\}
. \" simple accents for nroff and troff
.if n \{\
. ds ' \&
. ds ` \&
. ds ^ \&
. ds , \&
. ds ~ ~
. ds /
.\}
.if t \{\
. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
. \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
. \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
. \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
. ds : e
. ds 8 ss
. ds o a
. ds d- d\h'-1'\(ga
. ds D- D\h'-1'\(hy
. ds th \o'bp'
. ds Th \o'LP'
. ds ae ae
. ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "PERLUNICOOK 1"
.TH PERLUNICOOK 1 "2019-10-24" "perl v5.30.2" "Perl Programmers Reference Guide"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
perlunicook \- cookbookish examples of handling Unicode in Perl
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
This manpage contains short recipes demonstrating how to handle common Unicode
operations in Perl, plus one complete program at the end. Any undeclared
variables in individual recipes are assumed to have a previous appropriate
value in them.
.SH "EXAMPLES"
.IX Header "EXAMPLES"
.SS "X 0: Standard preamble"
.IX Subsection "X 0: Standard preamble"
Unless otherwise notes, all examples below require this standard preamble
to work correctly, with the \f(CW\*(C`#!\*(C'\fR adjusted to work on your system:
.PP
.Vb 1
\& #!/usr/bin/env perl
\&
\& use utf8; # so literals and identifiers can be in UTF\-8
\& use v5.12; # or later to get "unicode_strings" feature
\& use strict; # quote strings, declare variables
\& use warnings; # on by default
\& use warnings qw(FATAL utf8); # fatalize encoding glitches
\& use open qw(:std :encoding(UTF\-8)); # undeclared streams in UTF\-8
\& use charnames qw(:full :short); # unneeded in v5.16
.Ve
.PP
This \fIdoes\fR make even Unix programmers \f(CW\*(C`binmode\*(C'\fR your binary streams,
or open them with \f(CW\*(C`:raw\*(C'\fR, but that's the only way to get at them
portably anyway.
.PP
\&\fB\s-1WARNING\s0\fR: \f(CW\*(C`use autodie\*(C'\fR (pre 2.26) and \f(CW\*(C`use open\*(C'\fR do not get along with each
other.
.SS "X 1: Generic Unicode-savvy filter"
.IX Subsection "X 1: Generic Unicode-savvy filter"
Always decompose on the way in, then recompose on the way out.
.PP
.Vb 1
\& use Unicode::Normalize;
\&
\& while (<>) {
\& $_ = NFD($_); # decompose + reorder canonically
\& ...
\& } continue {
\& print NFC($_); # recompose (where possible) + reorder canonically
\& }
.Ve
.SS "X 2: Fine-tuning Unicode warnings"
.IX Subsection "X 2: Fine-tuning Unicode warnings"
As of v5.14, Perl distinguishes three subclasses of \s-1UTFX8\s0 warnings.
.PP
.Vb 4
\& use v5.14; # subwarnings unavailable any earlier
\& no warnings "nonchar"; # the 66 forbidden non\-characters
\& no warnings "surrogate"; # UTF\-16/CESU\-8 nonsense
\& no warnings "non_unicode"; # for codepoints over 0x10_FFFF
.Ve
.SS "X 3: Declare source in utf8 for identifiers and literals"
.IX Subsection "X 3: Declare source in utf8 for identifiers and literals"
Without the all-critical \f(CW\*(C`use utf8\*(C'\fR declaration, putting \s-1UTFX8\s0 in your
literals and identifiers wonXt work right. If you used the standard
preamble just given above, this already happened. If you did, you can
do things like this:
.PP
.Vb 1
\& use utf8;
\&
\& my $measure = "A\*ongstro\*:m";
\& my @Xsoft = qw( cp852 cp1251 cp1252 );
\& my @XXXXXXXXX = qw( XXXX XXXXX );
\& my @X = qw( koi8\-f koi8\-u koi8\-r );
\& my $motto = "X X X"; # FAMILY, GROWING HEART, DROMEDARY CAMEL
.Ve
.PP
If you forget \f(CW\*(C`use utf8\*(C'\fR, high bytes will be misunderstood as
separate characters, and nothing will work right.
.SS "X 4: Characters and their numbers"
.IX Subsection "X 4: Characters and their numbers"
The \f(CW\*(C`ord\*(C'\fR and \f(CW\*(C`chr\*(C'\fR functions work transparently on all codepoints,
not just on \s-1ASCII\s0 alone X nor in fact, not even just on Unicode alone.
.PP
.Vb 3
\& # ASCII characters
\& ord("A")
\& chr(65)
\&
\& # characters from the Basic Multilingual Plane
\& ord("X")
\& chr(0x3A3)
\&
\& # beyond the BMP
\& ord("X") # MATHEMATICAL ITALIC SMALL N
\& chr(0x1D45B)
\&
\& # beyond Unicode! (up to MAXINT)
\& ord("\ex{20_0000}")
\& chr(0x20_0000)
.Ve
.SS "X 5: Unicode literals by character number"
.IX Subsection "X 5: Unicode literals by character number"
In an interpolated literal, whether a double-quoted string or a
regex, you may specify a character by its number using the
\&\f(CW\*(C`\ex{\f(CIHHHHHH\f(CW}\*(C'\fR escape.
.PP
.Vb 2
\& String: "\ex{3a3}"
\& Regex: /\ex{3a3}/
\&
\& String: "\ex{1d45b}"
\& Regex: /\ex{1d45b}/
\&
\& # even non\-BMP ranges in regex work fine
\& /[\ex{1D434}\-\ex{1D467}]/
.Ve
.SS "X 6: Get character name by number"
.IX Subsection "X 6: Get character name by number"
.Vb 2
\& use charnames ();
\& my $name = charnames::viacode(0x03A3);
.Ve
.SS "X 7: Get character number by name"
.IX Subsection "X 7: Get character number by name"
.Vb 2
\& use charnames ();
\& my $number = charnames::vianame("GREEK CAPITAL LETTER SIGMA");
.Ve
.SS "X 8: Unicode named characters"
.IX Subsection "X 8: Unicode named characters"
Use the \f(CW\*(C`\eN{\f(CIcharname\f(CW}\*(C'\fR notation to get the character
by that name for use in interpolated literals (double-quoted
strings and regexes). In v5.16, there is an implicit
.PP
.Vb 1
\& use charnames qw(:full :short);
.Ve
.PP
But prior to v5.16, you must be explicit about which set of charnames you
want. The \f(CW\*(C`:full\*(C'\fR names are the official Unicode character name, alias, or
sequence, which all share a namespace.
.PP
.Vb 1
\& use charnames qw(:full :short latin greek);
\&
\& "\eN{MATHEMATICAL ITALIC SMALL N}" # :full
\& "\eN{GREEK CAPITAL LETTER SIGMA}" # :full
.Ve
.PP
Anything else is a Perl-specific convenience abbreviation. Specify one or
more scripts by names if you want short names that are script-specific.
.PP
.Vb 3
\& "\eN{Greek:Sigma}" # :short
\& "\eN{ae}" # latin
\& "\eN{epsilon}" # greek
.Ve
.PP
The v5.16 release also supports a \f(CW\*(C`:loose\*(C'\fR import for loose matching of
character names, which works just like loose matching of property names:
that is, it disregards case, whitespace, and underscores:
.PP
.Vb 1
\& "\eN{euro sign}" # :loose (from v5.16)
.Ve
.SS "X 9: Unicode named sequences"
.IX Subsection "X 9: Unicode named sequences"
These look just like character names but return multiple codepoints.
Notice the \f(CW%vx\fR vector-print functionality in \f(CW\*(C`printf\*(C'\fR.
.PP
.Vb 4
\& use charnames qw(:full);
\& my $seq = "\eN{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}";
\& printf "U+%v04X\en", $seq;
\& U+0100.0300
.Ve
.SS "X 10: Custom named characters"
.IX Subsection "X 10: Custom named characters"
Use \f(CW\*(C`:alias\*(C'\fR to give your own lexically scoped nicknames to existing
characters, or even to give unnamed private-use characters useful names.
.PP
.Vb 4
\& use charnames ":full", ":alias" => {
\& ecute => "LATIN SMALL LETTER E WITH ACUTE",
\& "APPLE LOGO" => 0xF8FF, # private use character
\& };
\&
\& "\eN{ecute}"
\& "\eN{APPLE LOGO}"
.Ve
.SS "X 11: Names of \s-1CJK\s0 codepoints"
.IX Subsection "X 11: Names of CJK codepoints"
Sinograms like \s-1XXXX\s0 come back with character names of
\&\f(CW\*(C`CJK UNIFIED IDEOGRAPH\-6771\*(C'\fR and \f(CW\*(C`CJK UNIFIED IDEOGRAPH\-4EAC\*(C'\fR,
because their XnamesX vary. The \s-1CPAN\s0 \f(CW\*(C`Unicode::Unihan\*(C'\fR module
has a large database for decoding these (and a whole lot more), provided you
know how to understand its output.
.PP
.Vb 8
\& # cpan \-i Unicode::Unihan
\& use Unicode::Unihan;
\& my $str = "XX";
\& my $unhan = Unicode::Unihan\->new;
\& for my $lang (qw(Mandarin Cantonese Korean JapaneseOn JapaneseKun)) {
\& printf "CJK $str in %\-12s is ", $lang;
\& say $unhan\->$lang($str);
\& }
.Ve
.PP
prints:
.PP
.Vb 5
\& CJK XX in Mandarin is DONG1JING1
\& CJK XX in Cantonese is dung1ging1
\& CJK XX in Korean is TONGKYENG
\& CJK XX in JapaneseOn is TOUKYOU KEI KIN
\& CJK XX in JapaneseKun is HIGASHI AZUMAMIYAKO
.Ve
.PP
If you have a specific romanization scheme in mind,
use the specific module:
.PP
.Vb 5
\& # cpan \-i Lingua::JA::Romanize::Japanese
\& use Lingua::JA::Romanize::Japanese;
\& my $k2r = Lingua::JA::Romanize::Japanese\->new;
\& my $str = "XX";
\& say "Japanese for $str is ", $k2r\->chars($str);
.Ve
.PP
prints
.PP
.Vb 1
\& Japanese for XX is toukyou
.Ve
.SS "X 12: Explicit encode/decode"
.IX Subsection "X 12: Explicit encode/decode"
On rare occasion, such as a database read, you may be
given encoded text you need to decode.
.PP
.Vb 1
\& use Encode qw(encode decode);
\&
\& my $chars = decode("shiftjis", $bytes, 1);
\& # OR
\& my $bytes = encode("MIME\-Header\-ISO_2022_JP", $chars, 1);
.Ve
.PP
For streams all in the same encoding, don't use encode/decode; instead
set the file encoding when you open the file or immediately after with
\&\f(CW\*(C`binmode\*(C'\fR as described later below.
.SS "X 13: Decode program arguments as utf8"
.IX Subsection "X 13: Decode program arguments as utf8"
.Vb 6
\& $ perl \-CA ...
\& or
\& $ export PERL_UNICODE=A
\& or
\& use Encode qw(decode);
\& @ARGV = map { decode(\*(AqUTF\-8\*(Aq, $_, 1) } @ARGV;
.Ve
.SS "X 14: Decode program arguments as locale encoding"
.IX Subsection "X 14: Decode program arguments as locale encoding"
.Vb 3
\& # cpan \-i Encode::Locale
\& use Encode qw(locale);
\& use Encode::Locale;
\&
\& # use "locale" as an arg to encode/decode
\& @ARGV = map { decode(locale => $_, 1) } @ARGV;
.Ve
.SS "X 15: Declare STD{\s-1IN,OUT,ERR\s0} to be utf8"
.IX Subsection "X 15: Declare STD{IN,OUT,ERR} to be utf8"
Use a command-line option, an environment variable, or else
call \f(CW\*(C`binmode\*(C'\fR explicitly:
.PP
.Vb 9
\& $ perl \-CS ...
\& or
\& $ export PERL_UNICODE=S
\& or
\& use open qw(:std :encoding(UTF\-8));
\& or
\& binmode(STDIN, ":encoding(UTF\-8)");
\& binmode(STDOUT, ":utf8");
\& binmode(STDERR, ":utf8");
.Ve
.SS "X 16: Declare STD{\s-1IN,OUT,ERR\s0} to be in locale encoding"
.IX Subsection "X 16: Declare STD{IN,OUT,ERR} to be in locale encoding"
.Vb 3
\& # cpan \-i Encode::Locale
\& use Encode;
\& use Encode::Locale;
\&
\& # or as a stream for binmode or open
\& binmode STDIN, ":encoding(console_in)" if \-t STDIN;
\& binmode STDOUT, ":encoding(console_out)" if \-t STDOUT;
\& binmode STDERR, ":encoding(console_out)" if \-t STDERR;
.Ve
.SS "X 17: Make file I/O default to utf8"
.IX Subsection "X 17: Make file I/O default to utf8"
Files opened without an encoding argument will be in \s-1UTF\-8:\s0
.PP
.Vb 5
\& $ perl \-CD ...
\& or
\& $ export PERL_UNICODE=D
\& or
\& use open qw(:encoding(UTF\-8));
.Ve
.SS "X 18: Make all I/O and args default to utf8"
.IX Subsection "X 18: Make all I/O and args default to utf8"
.Vb 7
\& $ perl \-CSDA ...
\& or
\& $ export PERL_UNICODE=SDA
\& or
\& use open qw(:std :encoding(UTF\-8));
\& use Encode qw(decode);
\& @ARGV = map { decode(\*(AqUTF\-8\*(Aq, $_, 1) } @ARGV;
.Ve
.SS "X 19: Open file with specific encoding"
.IX Subsection "X 19: Open file with specific encoding"
Specify stream encoding. This is the normal way
to deal with encoded text, not by calling low-level
functions.
.PP
.Vb 7
\& # input file
\& open(my $in_file, "< :encoding(UTF\-16)", "wintext");
\& OR
\& open(my $in_file, "<", "wintext");
\& binmode($in_file, ":encoding(UTF\-16)");
\& THEN
\& my $line = <$in_file>;
\&
\& # output file
\& open($out_file, "> :encoding(cp1252)", "wintext");
\& OR
\& open(my $out_file, ">", "wintext");
\& binmode($out_file, ":encoding(cp1252)");
\& THEN
\& print $out_file "some text\en";
.Ve
.PP
More layers than just the encoding can be specified here. For example,
the incantation \f(CW":raw :encoding(UTF\-16LE) :crlf"\fR includes implicit
\&\s-1CRLF\s0 handling.
.SS "X 20: Unicode casing"
.IX Subsection "X 20: Unicode casing"
Unicode casing is very different from \s-1ASCII\s0 casing.
.PP
.Vb 2
\& uc("henry X") # "HENRY X"
\& uc("tschu\*:\*8") # "TSCHU\*:SS" notice \*8 => SS
\&
\& # both are true:
\& "tschu\*:\*8" =~ /TSCHU\*:SS/i # notice \*8 => SS
\& "XXXXXXX" =~ /XXXXXXX/i # notice X,X,X sameness
.Ve
.SS "X 21: Unicode case-insensitive comparisons"
.IX Subsection "X 21: Unicode case-insensitive comparisons"
Also available in the \s-1CPAN\s0 Unicode::CaseFold module,
the new \f(CW\*(C`fc\*(C'\fR XfoldcaseX function from v5.16 grants
access to the same Unicode casefolding as the \f(CW\*(C`/i\*(C'\fR
pattern modifier has always used:
.PP
.Vb 1
\& use feature "fc"; # fc() function is from v5.16
\&
\& # sort case\-insensitively
\& my @sorted = sort { fc($a) cmp fc($b) } @list;
\&
\& # both are true:
\& fc("tschu\*:\*8") eq fc("TSCHU\*:SS")
\& fc("XXXXXXX") eq fc("XXXXXXX")
.Ve
.SS "X 22: Match Unicode linebreak sequence in regex"
.IX Subsection "X 22: Match Unicode linebreak sequence in regex"
A Unicode linebreak matches the two-character \s-1CRLF\s0
grapheme or any of seven vertical whitespace characters.
Good for dealing with textfiles coming from different
operating systems.
.PP
.Vb 1
\& \eR
\&
\& s/\eR/\en/g; # normalize all linebreaks to \en
.Ve
.SS "X 23: Get character category"
.IX Subsection "X 23: Get character category"
Find the general category of a numeric codepoint.
.PP
.Vb 2
\& use Unicode::UCD qw(charinfo);
\& my $cat = charinfo(0x3A3)\->{category}; # "Lu"
.Ve
.SS "X 24: Disabling Unicode-awareness in builtin charclasses"
.IX Subsection "X 24: Disabling Unicode-awareness in builtin charclasses"
Disable \f(CW\*(C`\ew\*(C'\fR, \f(CW\*(C`\eb\*(C'\fR, \f(CW\*(C`\es\*(C'\fR, \f(CW\*(C`\ed\*(C'\fR, and the \s-1POSIX\s0
classes from working correctly on Unicode either in this
scope, or in just one regex.
.PP
.Vb 2
\& use v5.14;
\& use re "/a";
\&
\& # OR
\&
\& my($num) = $str =~ /(\ed+)/a;
.Ve
.PP
Or use specific un-Unicode properties, like \f(CW\*(C`\ep{ahex}\*(C'\fR
and \f(CW\*(C`\ep{POSIX_Digit\*(C'\fR}. Properties still work normally
no matter what charset modifiers (\f(CW\*(C`/d /u /l /a /aa\*(C'\fR)
should be effect.
.SS "X 25: Match Unicode properties in regex with \ep, \eP"
.IX Subsection "X 25: Match Unicode properties in regex with p, P"
These all match a single codepoint with the given
property. Use \f(CW\*(C`\eP\*(C'\fR in place of \f(CW\*(C`\ep\*(C'\fR to match
one codepoint lacking that property.
.PP
.Vb 8
\& \epL, \epN, \epS, \epP, \epM, \epZ, \epC
\& \ep{Sk}, \ep{Ps}, \ep{Lt}
\& \ep{alpha}, \ep{upper}, \ep{lower}
\& \ep{Latin}, \ep{Greek}
\& \ep{script_extensions=Latin}, \ep{scx=Greek}
\& \ep{East_Asian_Width=Wide}, \ep{EA=W}
\& \ep{Line_Break=Hyphen}, \ep{LB=HY}
\& \ep{Numeric_Value=4}, \ep{NV=4}
.Ve
.SS "X 26: Custom character properties"
.IX Subsection "X 26: Custom character properties"
Define at compile-time your own custom character
properties for use in regexes.
.PP
.Vb 2
\& # using private\-use characters
\& sub In_Tengwar { "E000\etE07F\en" }
\&
\& if (/\ep{In_Tengwar}/) { ... }
\&
\& # blending existing properties
\& sub Is_GraecoRoman_Title {<<\*(AqEND_OF_SET\*(Aq}
\& +utf8::IsLatin
\& +utf8::IsGreek
\& &utf8::IsTitle
\& END_OF_SET
\&
\& if (/\ep{Is_GraecoRoman_Title}/ { ... }
.Ve
.SS "X 27: Unicode normalization"
.IX Subsection "X 27: Unicode normalization"
Typically render into \s-1NFD\s0 on input and \s-1NFC\s0 on output. Using \s-1NFKC\s0 or \s-1NFKD\s0
functions improves recall on searches, assuming you've already done to the
same text to be searched. Note that this is about much more than just pre\-
combined compatibility glyphs; it also reorders marks according to their
canonical combining classes and weeds out singletons.
.PP
.Vb 5
\& use Unicode::Normalize;
\& my $nfd = NFD($orig);
\& my $nfc = NFC($orig);
\& my $nfkd = NFKD($orig);
\& my $nfkc = NFKC($orig);
.Ve
.SS "X 28: Convert non-ASCII Unicode numerics"
.IX Subsection "X 28: Convert non-ASCII Unicode numerics"
Unless youXve used \f(CW\*(C`/a\*(C'\fR or \f(CW\*(C`/aa\*(C'\fR, \f(CW\*(C`\ed\*(C'\fR matches more than
\&\s-1ASCII\s0 digits only, but PerlXs implicit string-to-number
conversion does not current recognize these. HereXs how to
convert such strings manually.
.PP
.Vb 8
\& use v5.14; # needed for num() function
\& use Unicode::UCD qw(num);
\& my $str = "got X and XXXX and X and here";
\& my @nums = ();
\& while ($str =~ /(\ed+|\eN)/g) { # not just ASCII!
\& push @nums, num($1);
\& }
\& say "@nums"; # 12 4567 0.875
\&
\& use charnames qw(:full);
\& my $nv = num("\eN{RUMI DIGIT ONE}\eN{RUMI DIGIT TWO}");
.Ve
.SS "X 29: Match Unicode grapheme cluster in regex"
.IX Subsection "X 29: Match Unicode grapheme cluster in regex"
Programmer-visible XcharactersX are codepoints matched by \f(CW\*(C`/./s\*(C'\fR,
but user-visible XcharactersX are graphemes matched by \f(CW\*(C`/\eX/\*(C'\fR.
.PP
.Vb 3
\& # Find vowel *plus* any combining diacritics,underlining,etc.
\& my $nfd = NFD($orig);
\& $nfd =~ / (?=[aeiou]) \eX /xi
.Ve
.SS "X 30: Extract by grapheme instead of by codepoint (regex)"
.IX Subsection "X 30: Extract by grapheme instead of by codepoint (regex)"
.Vb 2
\& # match and grab five first graphemes
\& my($first_five) = $str =~ /^ ( \eX{5} ) /x;
.Ve
.SS "X 31: Extract by grapheme instead of by codepoint (substr)"
.IX Subsection "X 31: Extract by grapheme instead of by codepoint (substr)"
.Vb 4
\& # cpan \-i Unicode::GCString
\& use Unicode::GCString;
\& my $gcs = Unicode::GCString\->new($str);
\& my $first_five = $gcs\->substr(0, 5);
.Ve
.SS "X 32: Reverse string by grapheme"
.IX Subsection "X 32: Reverse string by grapheme"
Reversing by codepoint messes up diacritics, mistakenly converting
\&\f(CW\*(C`cre\*`me bru\*^le\*'e\*(C'\fR into \f(CW\*(C`e\*'elXurb emXerc\*(C'\fR instead of into \f(CW\*(C`ee\*'lu\*^rb eme\*`rc\*(C'\fR;
so reverse by grapheme instead. Both these approaches work
right no matter what normalization the string is in:
.PP
.Vb 1
\& $str = join("", reverse $str =~ /\eX/g);
\&
\& # OR: cpan \-i Unicode::GCString
\& use Unicode::GCString;
\& $str = reverse Unicode::GCString\->new($str);
.Ve
.SS "X 33: String length in graphemes"
.IX Subsection "X 33: String length in graphemes"
The string \f(CW\*(C`bru\*^le\*'e\*(C'\fR has six graphemes but up to eight codepoints.
This counts by grapheme, not by codepoint:
.PP
.Vb 3
\& my $str = "bru\*^le\*'e";
\& my $count = 0;
\& while ($str =~ /\eX/g) { $count++ }
\&
\& # OR: cpan \-i Unicode::GCString
\& use Unicode::GCString;
\& my $gcs = Unicode::GCString\->new($str);
\& my $count = $gcs\->length;
.Ve
.SS "X 34: Unicode column-width for printing"
.IX Subsection "X 34: Unicode column-width for printing"
PerlXs \f(CW\*(C`printf\*(C'\fR, \f(CW\*(C`sprintf\*(C'\fR, and \f(CW\*(C`format\*(C'\fR think all
codepoints take up 1 print column, but many take 0 or 2.
Here to show that normalization makes no difference,
we print out both forms:
.PP
.Vb 2
\& use Unicode::GCString;
\& use Unicode::Normalize;
\&
\& my @words = qw/cre\*`me bru\*^le\*'e/;
\& @words = map { NFC($_), NFD($_) } @words;
\&
\& for my $str (@words) {
\& my $gcs = Unicode::GCString\->new($str);
\& my $cols = $gcs\->columns;
\& my $pad = " " x (10 \- $cols);
\& say str, $pad, " |";
\& }
.Ve
.PP
generates this to show that it pads correctly no matter
the normalization:
.PP
.Vb 4
\& cre\*`me |
\& creXme |
\& bru\*^le\*'e |
\& bruXleXe |
.Ve
.SS "X 35: Unicode collation"
.IX Subsection "X 35: Unicode collation"
Text sorted by numeric codepoint follows no reasonable alphabetic order;
use the \s-1UCA\s0 for sorting text.
.PP
.Vb 3
\& use Unicode::Collate;
\& my $col = Unicode::Collate\->new();
\& my @list = $col\->sort(@old_list);
.Ve
.PP
See the \fIucsort\fR program from the Unicode::Tussle \s-1CPAN\s0 module
for a convenient command-line interface to this module.
.SS "X 36: Case\- \fIand\fP accent-insensitive Unicode sort"
.IX Subsection "X 36: Case- and accent-insensitive Unicode sort"
Specify a collation strength of level 1 to ignore case and
diacritics, only looking at the basic character.
.PP
.Vb 3
\& use Unicode::Collate;
\& my $col = Unicode::Collate\->new(level => 1);
\& my @list = $col\->sort(@old_list);
.Ve
.SS "X 37: Unicode locale collation"
.IX Subsection "X 37: Unicode locale collation"
Some locales have special sorting rules.
.PP
.Vb 4
\& # either use v5.12, OR: cpan \-i Unicode::Collate::Locale
\& use Unicode::Collate::Locale;
\& my $col = Unicode::Collate::Locale\->new(locale => "de_\|_phonebook");
\& my @list = $col\->sort(@old_list);
.Ve
.PP
The \fIucsort\fR program mentioned above accepts a \f(CW\*(C`\-\-locale\*(C'\fR parameter.
.ie n .SS "X 38: Making ""cmp"" work on text instead of codepoints"
.el .SS "X 38: Making \f(CWcmp\fP work on text instead of codepoints"
.IX Subsection "X 38: Making cmp work on text instead of codepoints"
Instead of this:
.PP
.Vb 5
\& @srecs = sort {
\& $b\->{AGE} <=> $a\->{AGE}
\& ||
\& $a\->{NAME} cmp $b\->{NAME}
\& } @recs;
.Ve
.PP
Use this:
.PP
.Vb 9
\& my $coll = Unicode::Collate\->new();
\& for my $rec (@recs) {
\& $rec\->{NAME_key} = $coll\->getSortKey( $rec\->{NAME} );
\& }
\& @srecs = sort {
\& $b\->{AGE} <=> $a\->{AGE}
\& ||
\& $a\->{NAME_key} cmp $b\->{NAME_key}
\& } @recs;
.Ve
.SS "X 39: Case\- \fIand\fP accent-insensitive comparisons"
.IX Subsection "X 39: Case- and accent-insensitive comparisons"
Use a collator object to compare Unicode text by character
instead of by codepoint.
.PP
.Vb 5
\& use Unicode::Collate;
\& my $es = Unicode::Collate\->new(
\& level => 1,
\& normalization => undef
\& );
\&
\& # now both are true:
\& $es\->eq("Garci\*'a", "GARCIA" );
\& $es\->eq("Ma\*'rquez", "MARQUEZ");
.Ve
.SS "X 40: Case\- \fIand\fP accent-insensitive locale comparisons"
.IX Subsection "X 40: Case- and accent-insensitive locale comparisons"
Same, but in a specific locale.
.PP
.Vb 3
\& my $de = Unicode::Collate::Locale\->new(
\& locale => "de_\|_phonebook",
\& );
\&
\& # now this is true:
\& $de\->eq("tschu\*:\*8", "TSCHUESS"); # notice u\*: => UE, \*8 => SS
.Ve
.SS "X 41: Unicode linebreaking"
.IX Subsection "X 41: Unicode linebreaking"
Break up text into lines according to Unicode rules.
.PP
.Vb 3
\& # cpan \-i Unicode::LineBreak
\& use Unicode::LineBreak;
\& use charnames qw(:full);
\&
\& my $para = "This is a super\eN{HYPHEN}long string. " x 20;
\& my $fmt = Unicode::LineBreak\->new;
\& print $fmt\->break($para), "\en";
.Ve
.SS "X 42: Unicode text in \s-1DBM\s0 hashes, the tedious way"
.IX Subsection "X 42: Unicode text in DBM hashes, the tedious way"
Using a regular Perl string as a key or value for a \s-1DBM\s0
hash will trigger a wide character exception if any codepoints
wonXt fit into a byte. HereXs how to manually manage the translation:
.PP
.Vb 3
\& use DB_File;
\& use Encode qw(encode decode);
\& tie %dbhash, "DB_File", "pathname";
\&
\& # STORE
\&
\& # assume $uni_key and $uni_value are abstract Unicode strings
\& my $enc_key = encode("UTF\-8", $uni_key, 1);
\& my $enc_value = encode("UTF\-8", $uni_value, 1);
\& $dbhash{$enc_key} = $enc_value;
\&
\& # FETCH
\&
\& # assume $uni_key holds a normal Perl string (abstract Unicode)
\& my $enc_key = encode("UTF\-8", $uni_key, 1);
\& my $enc_value = $dbhash{$enc_key};
\& my $uni_value = decode("UTF\-8", $enc_value, 1);
.Ve
.SS "X 43: Unicode text in \s-1DBM\s0 hashes, the easy way"
.IX Subsection "X 43: Unicode text in DBM hashes, the easy way"
HereXs how to implicitly manage the translation; all encoding
and decoding is done automatically, just as with streams that
have a particular encoding attached to them:
.PP
.Vb 2
\& use DB_File;
\& use DBM_Filter;
\&
\& my $dbobj = tie %dbhash, "DB_File", "pathname";
\& $dbobj\->Filter_Value("utf8"); # this is the magic bit
\&
\& # STORE
\&
\& # assume $uni_key and $uni_value are abstract Unicode strings
\& $dbhash{$uni_key} = $uni_value;
\&
\& # FETCH
\&
\& # $uni_key holds a normal Perl string (abstract Unicode)
\& my $uni_value = $dbhash{$uni_key};
.Ve
.SS "X 44: \s-1PROGRAM:\s0 Demo of Unicode collation and printing"
.IX Subsection "X 44: PROGRAM: Demo of Unicode collation and printing"
HereXs a full program showing how to make use of locale-sensitive
sorting, Unicode casing, and managing print widths when some of the
characters take up zero or two columns, not just one column each time.
When run, the following program produces this nicely aligned output:
.PP
.Vb 10
\& Cre\*`me Bru\*^le\*'e....... X2.00
\& E\*'clair............. X1.60
\& Fideua\*`............. X4.20
\& Hamburger.......... X6.00
\& Jamo\*'n Serrano...... X4.45
\& Linguic\*,a........... X7.00
\& Pa\*^te\*'............... X4.15
\& Pears.............. X2.00
\& Pe\*^ches............. X2.25
\& Smo\*/rbro\*/d........... X5.75
\& Spa\*:tzle............ X5.50
\& Xoric\*,o............. X3.00
\& XXXXX.............. X6.50
\& XXX............. X4.00
\& XXX............. X2.65
\& XXXXX......... X8.00
\& XXXXXXX..... X1.85
\& XX............... X9.99
\& XX............... X7.50
.Ve
.PP
Here's that program; tested on v5.14.
.PP
.Vb 12
\& #!/usr/bin/env perl
\& # umenu \- demo sorting and printing of Unicode food
\& #
\& # (obligatory and increasingly long preamble)
\& #
\& use utf8;
\& use v5.14; # for locale sorting
\& use strict;
\& use warnings;
\& use warnings qw(FATAL utf8); # fatalize encoding faults
\& use open qw(:std :encoding(UTF\-8)); # undeclared streams in UTF\-8
\& use charnames qw(:full :short); # unneeded in v5.16
\&
\& # std modules
\& use Unicode::Normalize; # std perl distro as of v5.8
\& use List::Util qw(max); # std perl distro as of v5.10
\& use Unicode::Collate::Locale; # std perl distro as of v5.14
\&
\& # cpan modules
\& use Unicode::GCString; # from CPAN
\&
\& # forward defs
\& sub pad($$$);
\& sub colwidth(_);
\& sub entitle(_);
\&
\& my %price = (
\& "XXXXX" => 6.50, # gyros
\& "pears" => 2.00, # like um, pears
\& "linguic\*,a" => 7.00, # spicy sausage, Portuguese
\& "xoric\*,o" => 3.00, # chorizo sausage, Catalan
\& "hamburger" => 6.00, # burgermeister meisterburger
\& "e\*'clair" => 1.60, # dessert, French
\& "smo\*/rbro\*/d" => 5.75, # sandwiches, Norwegian
\& "spa\*:tzle" => 5.50, # Bayerisch noodles, little sparrows
\& "XX" => 7.50, # bao1 zi5, steamed pork buns, Mandarin
\& "jamo\*'n serrano" => 4.45, # country ham, Spanish
\& "pe\*^ches" => 2.25, # peaches, French
\& "XXXXXXX" => 1.85, # cream\-filled pastry like eclair
\& "XXX" => 4.00, # makgeolli, Korean rice wine
\& "XX" => 9.99, # sushi, Japanese
\& "XXX" => 2.65, # omochi, rice cakes, Japanese
\& "cre\*`me bru\*^le\*'e" => 2.00, # crema catalana
\& "fideua\*`" => 4.20, # more noodles, Valencian
\& # (Catalan=fideuada)
\& "pa\*^te\*'" => 4.15, # gooseliver paste, French
\& "XXXXX" => 8.00, # okonomiyaki, Japanese
\& );
\&
\& my $width = 5 + max map { colwidth } keys %price;
\&
\& # So the Asian stuff comes out in an order that someone
\& # who reads those scripts won\*(Aqt freak out over; the
\& # CJK stuff will be in JIS X 0208 order that way.
\& my $coll = Unicode::Collate::Locale\->new(locale => "ja");
\&
\& for my $item ($coll\->sort(keys %price)) {
\& print pad(entitle($item), $width, ".");
\& printf " X%.2f\en", $price{$item};
\& }
\&
\& sub pad($$$) {
\& my($str, $width, $padchar) = @_;
\& return $str . ($padchar x ($width \- colwidth($str)));
\& }
\&
\& sub colwidth(_) {
\& my($str) = @_;
\& return Unicode::GCString\->new($str)\->columns;
\& }
\&
\& sub entitle(_) {
\& my($str) = @_;
\& $str =~ s{ (?=\epL)(\eS) (\eS*) }
\& { ucfirst($1) . lc($2) }xge;
\& return $str;
\& }
.Ve
.SH "SEE ALSO"
.IX Header "SEE ALSO"
See these manpages, some of which are \s-1CPAN\s0 modules:
perlunicode, perluniprops,
perlre, perlrecharclass,
perluniintro, perlunitut, perlunifaq,
PerlIO, DB_File, DBM_Filter, DBM_Filter::utf8,
Encode, Encode::Locale,
Unicode::UCD,
Unicode::Normalize,
Unicode::GCString, Unicode::LineBreak,
Unicode::Collate, Unicode::Collate::Locale,
Unicode::Unihan,
Unicode::CaseFold,
Unicode::Tussle,
Lingua::JA::Romanize::Japanese,
Lingua::ZH::Romanize::Pinyin,
Lingua::KO::Romanize::Hangul.
.PP
The Unicode::Tussle \s-1CPAN\s0 module includes many programs
to help with working with Unicode, including
these programs to fully or partly replace standard utilities:
\&\fItcgrep\fR instead of \fIegrep\fR,
\&\fIuniquote\fR instead of \fIcat \-v\fR or \fIhexdump\fR,
\&\fIuniwc\fR instead of \fIwc\fR,
\&\fIunilook\fR instead of \fIlook\fR,
\&\fIunifmt\fR instead of \fIfmt\fR,
and
\&\fIucsort\fR instead of \fIsort\fR.
For exploring Unicode character names and character properties,
see its \fIuniprops\fR, \fIunichars\fR, and \fIuninames\fR programs.
It also supplies these programs, all of which are general filters that do Unicode-y things:
\&\fIunititle\fR and \fIunicaps\fR;
\&\fIuniwide\fR and \fIuninarrow\fR;
\&\fIunisupers\fR and \fIunisubs\fR;
\&\fInfd\fR, \fInfc\fR, \fInfkd\fR, and \fInfkc\fR;
and \fIuc\fR, \fIlc\fR, and \fItc\fR.
.PP
Finally, see the published Unicode Standard (page numbers are from version
6.0.0), including these specific annexes and technical reports:
.IP "X3.13 Default Case Algorithms, page 113; X4.2 Case, pages 120X122; Case Mappings, page 166X172, especially Caseless Matching starting on page 170." 4
.IX Item "X3.13 Default Case Algorithms, page 113; X4.2 Case, pages 120X122; Case Mappings, page 166X172, especially Caseless Matching starting on page 170."
.PD 0
.IP "\s-1UAX\s0 #44: Unicode Character Database" 4
.IX Item "UAX #44: Unicode Character Database"
.IP "\s-1UTS\s0 #18: Unicode Regular Expressions" 4
.IX Item "UTS #18: Unicode Regular Expressions"
.IP "\s-1UAX\s0 #15: Unicode Normalization Forms" 4
.IX Item "UAX #15: Unicode Normalization Forms"
.IP "\s-1UTS\s0 #10: Unicode Collation Algorithm" 4
.IX Item "UTS #10: Unicode Collation Algorithm"
.IP "\s-1UAX\s0 #29: Unicode Text Segmentation" 4
.IX Item "UAX #29: Unicode Text Segmentation"
.IP "\s-1UAX\s0 #14: Unicode Line Breaking Algorithm" 4
.IX Item "UAX #14: Unicode Line Breaking Algorithm"
.IP "\s-1UAX\s0 #11: East Asian Width" 4
.IX Item "UAX #11: East Asian Width"
.PD
.SH "AUTHOR"
.IX Header "AUTHOR"
Tom Christiansen <tchrist@perl.com> wrote this, with occasional
kibbitzing from Larry Wall and Jeffrey Friedl in the background.
.SH "COPYRIGHT AND LICENCE"
.IX Header "COPYRIGHT AND LICENCE"
Copyright X 2012 Tom Christiansen.
.PP
This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.
.PP
Most of these examples taken from the current edition of the XCamel BookX;
that is, from the 4XX Edition of \fIProgramming Perl\fR, Copyright X 2012 Tom
Christiansen <et al.>, 2012\-02\-13 by OXReilly Media. The code itself is
freely redistributable, and you are encouraged to transplant, fold,
spindle, and mutilate any of the examples in this manpage however you please
for inclusion into your own programs without any encumbrance whatsoever.
Acknowledgement via code comment is polite but not required.
.SH "REVISION HISTORY"
.IX Header "REVISION HISTORY"
v1.0.0 X first public release, 2012\-02\-27