#!/usr/bin/env perl
#
# pagesource.pl - fetch a web page (or read a local HTML file), save its
# source, and print an analysis report.
#
# Core modules only: HTTP::Tiny, Getopt::Long, Digest::MD5, JSON::PP,
# Encode, Time::HiRes.
#
# Examples:
# perl pagesource.pl user
# perl pagesource.pl --url dx --out page.html --links --top 15
# perl pagesource.pl --file page.html --base https://example.com/docs/ --json
#
use strict;
use warnings;
use feature 'say';
use Getopt::Long qw(GetOptions);
use HTTP::Tiny;
use Digest::MD5 qw(md5_hex);
use JSON::PP;
use Encode qw(decode);
use Time::HiRes qw(time sleep);
binmode STDOUT, ':encoding(UTF-8)';
binmode STDERR, ':encoding(UTF-8)';
my $ATTR = qr/(?:"[^"]*"|'[^']*'|[^>"'])/;
# --------------------------------------------------------------------------
# Command line
# --------------------------------------------------------------------------
my %opt = (
agent => 'Mozilla/5.0 (X11; Linux x86_64) pagesource.pl/1.0',
timeout => 15,
retries => 3,
top => 10,
links => 0,
);
GetOptions(
\%opt, 'url=s', 'file=s', 'out=s', 'agent=s', 'timeout=i', 'retries=i',
'top=i', 'links!', 'json', 'print-source|print', 'base=s', 'help|h',
) or usage(2);
usage(0) if $opt{help};
$opt{url} //= shift @ARGV;
usage(2, 'Give a URL or --file.') unless $opt{url} || $opt{file};
usage(2, 'Use either a URL or --file, not both.') if $opt{url} && $opt{file};
if ($opt{url} && $opt{url} !~ m{^[a-z][a-z0-9+.-]*://}i) {
$opt{url} = "https://$opt{url}";
}
# --------------------------------------------------------------------------
# Get the HTML
# --------------------------------------------------------------------------
my ($html, $meta);
if ($opt{file}) {
($html, $meta) = read_local($opt{file});
} else {
($html, $meta) = fetch_url($opt{url}, \%opt);
}
if ($opt{out}) {
open my $fh, '>:encoding(UTF-8)', $opt{out}
or die "Cannot write $opt{out}: $!\n";
print {$fh} $html;
close $fh;
warn "Saved source to $opt{out}\n";
}
print $html, "\n" if $opt{'print-source'};
# --------------------------------------------------------------------------
# Analyze and report
# --------------------------------------------------------------------------
my $base = $opt{base} // ($opt{url} ? $meta->{final_url} // $opt{url} : undef);
my $info = analyze($html, $base);
$info->{fetch} = $meta;
if ($opt{json}) {
say JSON::PP->new->utf8(0)->pretty->canonical->encode($info);
} else {
print_report($info, \%opt);
}
exit 0;
# ==========================================================================
# Fetching
# ==========================================================================
sub read_local {
my ($path) = @_;
open my $fh, '<:raw', $path or die "Cannot read $path: $!\n";
local $/;
my $raw = <$fh>;
close $fh;
return (decode('UTF-8', $raw, Encode::FB_DEFAULT),
{ source => "file:$path", bytes => length $raw });
}
sub fetch_url {
my ($url, $o) = @_;
my $http = HTTP::Tiny->new(
agent => $o->{agent},
timeout => $o->{timeout},
max_redirect => 5,
verify_SSL => 1,
);
my ($res, $t0);
for my $attempt (1 .. $o->{retries}) {
$t0 = time;
$res = $http->get($url, { headers => { 'Accept-Encoding' => 'identity' } });
last if $res->{success};
my $status = $res->{status};
my $retryable = $status >= 500; # 599 = connection-level failure
my $why = $status == 599 ? ($res->{content} // 'connection error')
: "HTTP $status $res->{reason}";
chomp $why;
if (!$retryable || $attempt == $o->{retries}) {
die "Failed to fetch $url: $why\n";
}
my $wait = 2**($attempt - 1);
warn "Attempt $attempt failed ($why); retrying in ${wait}s...\n";
sleep $wait;
}
my $elapsed = time - $t0;
my $ctype = $res->{headers}{'content-type'} // '';
my $charset = $ctype =~ /charset=["']?([\w-]+)/i ? $1 : 'UTF-8';
my $text = eval { decode($charset, $res->{content}, Encode::FB_DEFAULT) }
// decode('UTF-8', $res->{content}, Encode::FB_DEFAULT);
return ($text, {
source => $url,
final_url => $res->{url},
status => $res->{status},
content_type => $ctype,
server => $res->{headers}{server} // '',
bytes => length $res->{content},
seconds => sprintf('%.2f', $elapsed),
});
}
# ==========================================================================
# Analysis
# ==========================================================================
sub analyze {
my ($html, $base) = @_;
my %r = (
bytes_decoded => length $html,
md5 => md5_hex(do { my $c = $html; utf8::encode($c); $c }),
lines => scalar(() = $html =~ /\n/g) + 1,
);
# Strip comments, and the bodies of <script>/<style>, before analysis.
my $clean = $html;
$r{comments} = () = $clean =~ /<!--.*?-->/gs;
$clean =~ s/<!--.*?-->//gs;
$r{inline_scripts} = $r{inline_script_bytes} = 0;
$clean =~ s{(<script\b$ATTR*>)(.*?)</script\s*>}{
my ($open, $body) = ($1, $2);
if ($body =~ /\S/) {
$r{inline_scripts}++;
$r{inline_script_bytes} += length $body;
}
"$open</script>"
}gise;
$r{inline_styles} = 0;
$clean =~ s{(<style\b$ATTR*>)(.*?)</style\s*>}{
my ($open, $body) = ($1, $2);
$r{inline_styles}++ if $body =~ /\S/;
"$open</style>"
}gise;
# Walk every tag once.
my (%tags, %meta, @hrefs, @imgs, @scripts, @styles);
while ($clean =~ /<([a-zA-Z][a-zA-Z0-9-]*)\b($ATTR*)>/g) {
my ($tag, $attrs) = (lc $1, $2);
$tags{$tag}++;
my %a = parse_attrs($attrs);
if ($tag eq 'a' && defined $a{href}) { push @hrefs, $a{href} }
elsif ($tag eq 'img' && defined $a{src}) { push @imgs, $a{src} }
elsif ($tag eq 'script' && defined $a{src}) { push @scripts, $a{src} }
elsif ($tag eq 'link' && defined $a{href}
&& ($a{rel} // '') =~ /stylesheet/i) { push @styles, $a{href} }
elsif ($tag eq 'html' && defined $a{lang}) { $r{lang} = $a{lang} }
elsif ($tag eq 'meta') {
$meta{charset} = $a{charset} if defined $a{charset};
my $key = $a{name} // $a{property} // $a{'http-equiv'};
$meta{lc $key} = $a{content} if defined $key && defined $a{content};
}
}
$r{tags} = \%tags;
$r{tag_total} = 0;
$r{tag_total} += $_ for values %tags;
$r{meta} = \%meta;
# Title
if ($clean =~ m{<title\b[^>]*>(.*?)</title>}is) {
$r{title} = squash(strip_tags($1));
}
# Heading outline
my @outline;
while ($clean =~ m{<(h[1-6])\b$ATTR*>(.*?)</\1\s*>}gis) {
my $text = squash(strip_tags($2));
push @outline, { level => substr(lc $1, 1), text => $text } if length $text;
}
$r{outline} = \@outline;
# Visible text
my $text = squash(strip_tags($clean));
$r{text_chars} = length $text;
$r{words} = () = $text =~ /\w+/g;
# Links
my $host = host_of($base);
my (%seen, @internal, @external, @other);
for my $href (@hrefs) {
$href =~ s/^\s+|\s+$//g;
next if $href eq '' || $href =~ /^#/;
if ($href =~ /^(?:mailto|tel|javascript|data):/i) {
push @other, $href;
next;
}
my $abs = resolve_url($base, $href) // $href;
$abs =~ s/#.*//;
next if $seen{$abs}++;
my $h = host_of($abs);
if ($h ne '' && $host ne '' && $h ne $host) { push @external, $abs }
elsif ($h ne '') { push @internal, $abs }
else { push @other, $abs }
}
$r{links} = {
internal => \@internal,
external => \@external,
other => \@other,
};
$r{resources} = {
images => [ map { resolve_url($base, $_) // $_ } @imgs ],
scripts => [ map { resolve_url($base, $_) // $_ } @scripts ],
stylesheets => [ map { resolve_url($base, $_) // $_ } @styles ],
};
# External domains ranked by link count.
my %domains;
$domains{ host_of($_) }++ for @external;
$r{top_external_domains} = [
map { { domain => $_, links => $domains{$_} } }
sort { $domains{$b} <=> $domains{$a} || $a cmp $b } keys %domains
];
return \%r;
}
sub parse_attrs {
my ($s) = @_;
my %a;
while ($s =~ /([a-zA-Z_:][-\w:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g) {
my $name = lc $1;
my $val = $2 // $3 // $4 // '';
$a{$name} //= decode_entities($val);
}
return %a;
}
sub strip_tags { my ($s) = @_; $s =~ s/<[^>]*>/ /g; return decode_entities($s) }
sub squash { my ($s) = @_; $s =~ s/\s+/ /g; $s =~ s/^ | $//g; return $s }
sub decode_entities {
my ($s) = @_;
my %named = (amp => '&', lt => '<', gt => '>', quot => '"',
apos => "'", nbsp => ' ', copy => "\x{A9}", mdash => "\x{2014}");
$s =~ s{&(?:#(\d+)|#x([0-9a-fA-F]+)|(\w+));}{
defined $1 ? chr($1)
: defined $2 ? chr(hex $2)
: exists $named{lc $3} ? $named{lc $3}
: "&$3;"
}ge;
return $s;
}
sub host_of {
my ($url) = @_;
return '' unless defined $url && $url =~ m{^https?://([^/:?#]+)}i;
(my $h = lc $1) =~ s/^www\.//;
return $h;
}
sub resolve_url {
my ($base, $href) = @_;
return $href if $href =~ m{^[a-z][a-z0-9+.-]*:}i;
return undef unless defined $base
&& $base =~ m{^(https?:)//([^/?#]+)([^?#]*)}i;
my ($scheme, $auth, $path) = ($1, $2, $3);
return "$scheme$href" if $href =~ m{^//};
return "$scheme//$auth$href" if $href =~ m{^/};
$path =~ s{[^/]*$}{};
$path = '/' if $path eq '';
my @out;
for my $seg (split m{/}, $path . $href, -1) {
if ($seg eq '..') { pop @out if @out > 1 }
elsif ($seg eq '.') { next }
else { push @out, $seg }
}
return "$scheme//$auth" . join('/', @out);
}
# ==========================================================================
# Reporting
# ==========================================================================
sub print_report {
my ($i, $o) = @_;
my $f = $i->{fetch};
say '=' x 64;
say 'PAGE SOURCE REPORT';
say '=' x 64;
printf "%-16s %s\n", 'Source:', $f->{source};
printf "%-16s %s\n", 'Final URL:', $f->{final_url} if $f->{final_url};
printf "%-16s %s\n", 'HTTP status:', $f->{status} if $f->{status};
printf "%-16s %s\n", 'Content-Type:', $f->{content_type} if $f->{content_type};
printf "%-16s %s\n", 'Server:', $f->{server} if $f->{server};
printf "%-16s %s bytes%s\n", 'Size:', commify($f->{bytes}),
$f->{seconds} ? " in $f->{seconds}s" : '';
printf "%-16s %s\n", 'Lines:', commify($i->{lines});
printf "%-16s %s\n", 'MD5:', $i->{md5};
printf "%-16s %s\n", 'Language:', $i->{lang} if $i->{lang};
printf "%-16s %s\n", 'Title:', $i->{title} // '(none)';
say '';
my $desc = $i->{meta}{description} // $i->{meta}{'og:description'};
if (defined $desc) {
$desc = substr($desc, 0, 197) . '...' if length $desc > 200;
say "Description: $desc";
say '';
}
say '-- Content ' . '-' x 52;
printf "Words: %s Characters of text: %s Comments: %d\n",
commify($i->{words}), commify($i->{text_chars}), $i->{comments};
printf "Inline scripts: %d (%s bytes) Inline styles: %d\n",
$i->{inline_scripts}, commify($i->{inline_script_bytes}), $i->{inline_styles};
say '';
say "-- Top $o->{top} tags (of " . commify($i->{tag_total}) . ') ' . '-' x 30;
my $tags = $i->{tags};
my @ranked = sort { $tags->{$b} <=> $tags->{$a} || $a cmp $b } keys %$tags;
@ranked = @ranked[0 .. $o->{top} - 1] if @ranked > $o->{top};
my $max = @ranked ? $tags->{$ranked[0]} : 1;
for my $t (@ranked) {
my $bar = '#' x int(30 * $tags->{$t} / $max + 0.5);
printf " %-10s %6s %s\n", "<$t>", commify($tags->{$t}), $bar;
}
say '';
if (@{ $i->{outline} }) {
say '-- Heading outline ' . '-' x 45;
for my $h (@{ $i->{outline} }[0 .. min(19, $#{ $i->{outline} })]) {
my $text = length $h->{text} > 70 ? substr($h->{text}, 0, 67) . '...' : $h->{text};
say ' ' . (' ' x ($h->{level} - 1)) . "h$h->{level} $text";
}
say ' ...' if @{ $i->{outline} } > 20;
say '';
}
my $l = $i->{links};
say '-- Links ' . '-' x 54;
printf "Internal: %d External: %d Other (mailto/tel/etc): %d\n",
scalar @{ $l->{internal} }, scalar @{ $l->{external} }, scalar @{ $l->{other} };
if (@{ $i->{top_external_domains} }) {
say 'Top external domains:';
my @d = @{ $i->{top_external_domains} };
@d = @d[0 .. $o->{top} - 1] if @d > $o->{top};
printf " %-36s %d\n", $_->{domain}, $_->{links} for @d;
}
if ($o->{links}) {
say 'All internal links:';
say " $_" for @{ $l->{internal} };
say 'All external links:';
say " $_" for @{ $l->{external} };
}
say '';
my $res = $i->{resources};
say '-- Resources ' . '-' x 50;
printf "Images: %d Scripts: %d Stylesheets: %d\n",
scalar @{ $res->{images} }, scalar @{ $res->{scripts} }, scalar @{ $res->{stylesheets} };
say '';
}
sub commify {
my ($n) = @_;
$n = 0 unless defined $n;
1 while $n =~ s/^(\d+)(\d{3})/$1,$2/;
return $n;
}
sub min { $_[0] < $_[1] ? $_[0] : $_[1] }
sub usage {
my ($code, $msg) = @_;
my $fh = $code ? \*STDERR : \*STDOUT;
print {$fh} "$msg\n\n" if $msg;
print {$fh} <<'USAGE';
Usage: pagesource.pl [options] [URL]
--url URL page to fetch (https:// is assumed if omitted)
--file PATH analyze a local HTML file instead of fetching
--base URL base URL used to resolve links when using --file
--out PATH save the raw source to PATH
--print also print the raw source to stdout
--links list every internal and external link
--top N rows in the tag/domain tables (default 10)
--json emit the analysis as JSON instead of a text report
--agent STRING User-Agent header
--timeout SECS network timeout (default 15)
--retries N attempts for 5xx/connection errors (default 3)
-h, --help show this help
USAGE
exit $code;
}