Friday, October 30, 2026

this is what im most most proud of 

import urllib.request

URL = "prg"

# Some sites reject the default Python user agent, so send a browser-like one.
req = urllib.request.Request(
    URL,
    headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
                           "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"},
)

with urllib.request.urlopen(req, timeout=15) as resp:
    html = resp.read().decode(resp.headers.get_content_charset() or "utf-8", errors="replace")

print(f"Status: {resp.status}, length: {len(html)} characters\n")
print(html)

# Optionally save it to a file:
with open("user", "w", encoding="utf-8") as f:
    f.write(html)


#!/usr/bin/env perl
#
# pagesource.pl - fetch a web page (or read a local HTML file), save its
# source, and print an analysis report.
#
# Core modules only: HTTP::Tiny, Getopt::Long, Digest::MD5, JSON::PP,
# Encode, Time::HiRes.
#
# Examples:
#   perl pagesource.pl user
#   perl pagesource.pl --url dx --out page.html --links --top 15
#   perl pagesource.pl --file page.html --base https://example.com/docs/ --json
#
use strict;
use warnings;
use feature 'say';
use Getopt::Long qw(GetOptions);
use HTTP::Tiny;
use Digest::MD5 qw(md5_hex);
use JSON::PP;
use Encode qw(decode);
use Time::HiRes qw(time sleep);

binmode STDOUT, ':encoding(UTF-8)';
binmode STDERR, ':encoding(UTF-8)';

my $ATTR = qr/(?:"[^"]*"|'[^']*'|[^>"'])/;

# --------------------------------------------------------------------------
# Command line
# --------------------------------------------------------------------------
my %opt = (
    agent   => 'Mozilla/5.0 (X11; Linux x86_64) pagesource.pl/1.0',
    timeout => 15,
    retries => 3,
    top     => 10,
    links   => 0,
);

GetOptions(
    \%opt, 'url=s', 'file=s', 'out=s', 'agent=s', 'timeout=i', 'retries=i',
    'top=i', 'links!', 'json', 'print-source|print', 'base=s', 'help|h',
) or usage(2);
usage(0) if $opt{help};

$opt{url} //= shift @ARGV;
usage(2, 'Give a URL or --file.') unless $opt{url} || $opt{file};
usage(2, 'Use either a URL or --file, not both.') if $opt{url} && $opt{file};

if ($opt{url} && $opt{url} !~ m{^[a-z][a-z0-9+.-]*://}i) {
    $opt{url} = "https://$opt{url}";
}

# --------------------------------------------------------------------------
# Get the HTML
# --------------------------------------------------------------------------
my ($html, $meta);
if ($opt{file}) {
    ($html, $meta) = read_local($opt{file});
} else {
    ($html, $meta) = fetch_url($opt{url}, \%opt);
}

if ($opt{out}) {
    open my $fh, '>:encoding(UTF-8)', $opt{out}
        or die "Cannot write $opt{out}: $!\n";
    print {$fh} $html;
    close $fh;
    warn "Saved source to $opt{out}\n";
}

print $html, "\n" if $opt{'print-source'};

# --------------------------------------------------------------------------
# Analyze and report
# --------------------------------------------------------------------------
my $base = $opt{base} // ($opt{url} ? $meta->{final_url} // $opt{url} : undef);
my $info = analyze($html, $base);
$info->{fetch} = $meta;

if ($opt{json}) {
    say JSON::PP->new->utf8(0)->pretty->canonical->encode($info);
} else {
    print_report($info, \%opt);
}
exit 0;

# ==========================================================================
# Fetching
# ==========================================================================
sub read_local {
    my ($path) = @_;
    open my $fh, '<:raw', $path or die "Cannot read $path: $!\n";
    local $/;
    my $raw = <$fh>;
    close $fh;
    return (decode('UTF-8', $raw, Encode::FB_DEFAULT),
            { source => "file:$path", bytes => length $raw });
}

sub fetch_url {
    my ($url, $o) = @_;
    my $http = HTTP::Tiny->new(
        agent        => $o->{agent},
        timeout      => $o->{timeout},
        max_redirect => 5,
        verify_SSL   => 1,
    );

    my ($res, $t0);
    for my $attempt (1 .. $o->{retries}) {
        $t0  = time;
        $res = $http->get($url, { headers => { 'Accept-Encoding' => 'identity' } });
        last if $res->{success};

        my $status = $res->{status};
        my $retryable = $status >= 500;    # 599 = connection-level failure
        my $why = $status == 599 ? ($res->{content} // 'connection error')
                                 : "HTTP $status $res->{reason}";
        chomp $why;
        if (!$retryable || $attempt == $o->{retries}) {
            die "Failed to fetch $url: $why\n";
        }
        my $wait = 2**($attempt - 1);
        warn "Attempt $attempt failed ($why); retrying in ${wait}s...\n";
        sleep $wait;
    }

    my $elapsed = time - $t0;
    my $ctype   = $res->{headers}{'content-type'} // '';
    my $charset = $ctype =~ /charset=["']?([\w-]+)/i ? $1 : 'UTF-8';
    my $text    = eval { decode($charset, $res->{content}, Encode::FB_DEFAULT) }
                  // decode('UTF-8', $res->{content}, Encode::FB_DEFAULT);

    return ($text, {
        source       => $url,
        final_url    => $res->{url},
        status       => $res->{status},
        content_type => $ctype,
        server       => $res->{headers}{server} // '',
        bytes        => length $res->{content},
        seconds      => sprintf('%.2f', $elapsed),
    });
}

# ==========================================================================
# Analysis
# ==========================================================================
sub analyze {
    my ($html, $base) = @_;
    my %r = (
        bytes_decoded => length $html,
        md5           => md5_hex(do { my $c = $html; utf8::encode($c); $c }),
        lines         => scalar(() = $html =~ /\n/g) + 1,
    );

    # Strip comments, and the bodies of <script>/<style>, before analysis.
    my $clean = $html;
    $r{comments} = () = $clean =~ /<!--.*?-->/gs;
    $clean =~ s/<!--.*?-->//gs;

    $r{inline_scripts} = $r{inline_script_bytes} = 0;
    $clean =~ s{(<script\b$ATTR*>)(.*?)</script\s*>}{
        my ($open, $body) = ($1, $2);
        if ($body =~ /\S/) {
            $r{inline_scripts}++;
            $r{inline_script_bytes} += length $body;
        }
        "$open</script>"
    }gise;

    $r{inline_styles} = 0;
    $clean =~ s{(<style\b$ATTR*>)(.*?)</style\s*>}{
        my ($open, $body) = ($1, $2);
        $r{inline_styles}++ if $body =~ /\S/;
        "$open</style>"
    }gise;

    # Walk every tag once.
    my (%tags, %meta, @hrefs, @imgs, @scripts, @styles);
    while ($clean =~ /<([a-zA-Z][a-zA-Z0-9-]*)\b($ATTR*)>/g) {
        my ($tag, $attrs) = (lc $1, $2);
        $tags{$tag}++;
        my %a = parse_attrs($attrs);

        if    ($tag eq 'a'      && defined $a{href}) { push @hrefs,   $a{href} }
        elsif ($tag eq 'img'    && defined $a{src})  { push @imgs,    $a{src}  }
        elsif ($tag eq 'script' && defined $a{src})  { push @scripts, $a{src}  }
        elsif ($tag eq 'link'   && defined $a{href}
              && ($a{rel} // '') =~ /stylesheet/i)   { push @styles,  $a{href} }
        elsif ($tag eq 'html'   && defined $a{lang}) { $r{lang} = $a{lang} }
        elsif ($tag eq 'meta') {
            $meta{charset} = $a{charset} if defined $a{charset};
            my $key = $a{name} // $a{property} // $a{'http-equiv'};
            $meta{lc $key} = $a{content} if defined $key && defined $a{content};
        }
    }
    $r{tags}      = \%tags;
    $r{tag_total} = 0;
    $r{tag_total} += $_ for values %tags;
    $r{meta}      = \%meta;

    # Title
    if ($clean =~ m{<title\b[^>]*>(.*?)</title>}is) {
        $r{title} = squash(strip_tags($1));
    }

    # Heading outline
    my @outline;
    while ($clean =~ m{<(h[1-6])\b$ATTR*>(.*?)</\1\s*>}gis) {
        my $text = squash(strip_tags($2));
        push @outline, { level => substr(lc $1, 1), text => $text } if length $text;
    }
    $r{outline} = \@outline;

    # Visible text
    my $text = squash(strip_tags($clean));
    $r{text_chars} = length $text;
    $r{words}      = () = $text =~ /\w+/g;

    # Links
    my $host = host_of($base);
    my (%seen, @internal, @external, @other);
    for my $href (@hrefs) {
        $href =~ s/^\s+|\s+$//g;
        next if $href eq '' || $href =~ /^#/;
        if ($href =~ /^(?:mailto|tel|javascript|data):/i) {
            push @other, $href;
            next;
        }
        my $abs = resolve_url($base, $href) // $href;
        $abs =~ s/#.*//;
        next if $seen{$abs}++;
        my $h = host_of($abs);
        if ($h ne '' && $host ne '' && $h ne $host) { push @external, $abs }
        elsif ($h ne '')                            { push @internal, $abs }
        else                                        { push @other,    $abs }
    }
    $r{links} = {
        internal => \@internal,
        external => \@external,
        other    => \@other,
    };
    $r{resources} = {
        images      => [ map { resolve_url($base, $_) // $_ } @imgs    ],
        scripts     => [ map { resolve_url($base, $_) // $_ } @scripts ],
        stylesheets => [ map { resolve_url($base, $_) // $_ } @styles  ],
    };

    # External domains ranked by link count.
    my %domains;
    $domains{ host_of($_) }++ for @external;
    $r{top_external_domains} = [
        map { { domain => $_, links => $domains{$_} } }
        sort { $domains{$b} <=> $domains{$a} || $a cmp $b } keys %domains
    ];

    return \%r;
}

sub parse_attrs {
    my ($s) = @_;
    my %a;
    while ($s =~ /([a-zA-Z_:][-\w:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g) {
        my $name = lc $1;
        my $val  = $2 // $3 // $4 // '';
        $a{$name} //= decode_entities($val);
    }
    return %a;
}

sub strip_tags { my ($s) = @_; $s =~ s/<[^>]*>/ /g; return decode_entities($s) }
sub squash     { my ($s) = @_; $s =~ s/\s+/ /g; $s =~ s/^ | $//g; return $s }

sub decode_entities {
    my ($s) = @_;
    my %named = (amp => '&', lt => '<', gt => '>', quot => '"',
                 apos => "'", nbsp => ' ', copy => "\x{A9}", mdash => "\x{2014}");
    $s =~ s{&(?:#(\d+)|#x([0-9a-fA-F]+)|(\w+));}{
        defined $1 ? chr($1)
      : defined $2 ? chr(hex $2)
      : exists $named{lc $3} ? $named{lc $3}
      : "&$3;"
    }ge;
    return $s;
}

sub host_of {
    my ($url) = @_;
    return '' unless defined $url && $url =~ m{^https?://([^/:?#]+)}i;
    (my $h = lc $1) =~ s/^www\.//;
    return $h;
}

sub resolve_url {
    my ($base, $href) = @_;
    return $href if $href =~ m{^[a-z][a-z0-9+.-]*:}i;
    return undef unless defined $base
        && $base =~ m{^(https?:)//([^/?#]+)([^?#]*)}i;
    my ($scheme, $auth, $path) = ($1, $2, $3);

    return "$scheme$href"        if $href =~ m{^//};
    return "$scheme//$auth$href" if $href =~ m{^/};

    $path =~ s{[^/]*$}{};
    $path = '/' if $path eq '';
    my @out;
    for my $seg (split m{/}, $path . $href, -1) {
        if    ($seg eq '..') { pop @out if @out > 1 }
        elsif ($seg eq '.')  { next }
        else                 { push @out, $seg }
    }
    return "$scheme//$auth" . join('/', @out);
}

# ==========================================================================
# Reporting
# ==========================================================================
sub print_report {
    my ($i, $o) = @_;
    my $f = $i->{fetch};

    say '=' x 64;
    say 'PAGE SOURCE REPORT';
    say '=' x 64;
    printf "%-16s %s\n", 'Source:',       $f->{source};
    printf "%-16s %s\n", 'Final URL:',    $f->{final_url}    if $f->{final_url};
    printf "%-16s %s\n", 'HTTP status:',  $f->{status}       if $f->{status};
    printf "%-16s %s\n", 'Content-Type:', $f->{content_type} if $f->{content_type};
    printf "%-16s %s\n", 'Server:',       $f->{server}       if $f->{server};
    printf "%-16s %s bytes%s\n", 'Size:', commify($f->{bytes}),
        $f->{seconds} ? " in $f->{seconds}s" : '';
    printf "%-16s %s\n", 'Lines:',        commify($i->{lines});
    printf "%-16s %s\n", 'MD5:',          $i->{md5};
    printf "%-16s %s\n", 'Language:',     $i->{lang} if $i->{lang};
    printf "%-16s %s\n", 'Title:',        $i->{title} // '(none)';
    say '';

    my $desc = $i->{meta}{description} // $i->{meta}{'og:description'};
    if (defined $desc) {
        $desc = substr($desc, 0, 197) . '...' if length $desc > 200;
        say "Description: $desc";
        say '';
    }

    say '-- Content ' . '-' x 52;
    printf "Words: %s   Characters of text: %s   Comments: %d\n",
        commify($i->{words}), commify($i->{text_chars}), $i->{comments};
    printf "Inline scripts: %d (%s bytes)   Inline styles: %d\n",
        $i->{inline_scripts}, commify($i->{inline_script_bytes}), $i->{inline_styles};
    say '';

    say "-- Top $o->{top} tags (of " . commify($i->{tag_total}) . ') ' . '-' x 30;
    my $tags = $i->{tags};
    my @ranked = sort { $tags->{$b} <=> $tags->{$a} || $a cmp $b } keys %$tags;
    @ranked = @ranked[0 .. $o->{top} - 1] if @ranked > $o->{top};
    my $max = @ranked ? $tags->{$ranked[0]} : 1;
    for my $t (@ranked) {
        my $bar = '#' x int(30 * $tags->{$t} / $max + 0.5);
        printf "  %-10s %6s  %s\n", "<$t>", commify($tags->{$t}), $bar;
    }
    say '';

    if (@{ $i->{outline} }) {
        say '-- Heading outline ' . '-' x 45;
        for my $h (@{ $i->{outline} }[0 .. min(19, $#{ $i->{outline} })]) {
            my $text = length $h->{text} > 70 ? substr($h->{text}, 0, 67) . '...' : $h->{text};
            say '  ' . ('  ' x ($h->{level} - 1)) . "h$h->{level} $text";
        }
        say '  ...' if @{ $i->{outline} } > 20;
        say '';
    }

    my $l = $i->{links};
    say '-- Links ' . '-' x 54;
    printf "Internal: %d   External: %d   Other (mailto/tel/etc): %d\n",
        scalar @{ $l->{internal} }, scalar @{ $l->{external} }, scalar @{ $l->{other} };
    if (@{ $i->{top_external_domains} }) {
        say 'Top external domains:';
        my @d = @{ $i->{top_external_domains} };
        @d = @d[0 .. $o->{top} - 1] if @d > $o->{top};
        printf "  %-36s %d\n", $_->{domain}, $_->{links} for @d;
    }
    if ($o->{links}) {
        say 'All internal links:';
        say "  $_" for @{ $l->{internal} };
        say 'All external links:';
        say "  $_" for @{ $l->{external} };
    }
    say '';

    my $res = $i->{resources};
    say '-- Resources ' . '-' x 50;
    printf "Images: %d   Scripts: %d   Stylesheets: %d\n",
        scalar @{ $res->{images} }, scalar @{ $res->{scripts} }, scalar @{ $res->{stylesheets} };
    say '';
}

sub commify {
    my ($n) = @_;
    $n = 0 unless defined $n;
    1 while $n =~ s/^(\d+)(\d{3})/$1,$2/;
    return $n;
}

sub min { $_[0] < $_[1] ? $_[0] : $_[1] }

sub usage {
    my ($code, $msg) = @_;
    my $fh = $code ? \*STDERR : \*STDOUT;
    print {$fh} "$msg\n\n" if $msg;
    print {$fh} <<'USAGE';
Usage: pagesource.pl [options] [URL]

  --url URL         page to fetch (https:// is assumed if omitted)
  --file PATH       analyze a local HTML file instead of fetching
  --base URL        base URL used to resolve links when using --file
  --out PATH        save the raw source to PATH
  --print           also print the raw source to stdout
  --links           list every internal and external link
  --top N           rows in the tag/domain tables (default 10)
  --json            emit the analysis as JSON instead of a text report
  --agent STRING    User-Agent header
  --timeout SECS    network timeout (default 15)
  --retries N       attempts for 5xx/connection errors (default 3)
  -h, --help        show this help
USAGE
    exit $code;
}
at

No comments:

Post a Comment

no thanks

Note: Only a member of this blog may post a comment.

Subscribe to: Post Comments (Atom)

this no one will understand i promise

  #!/usr/bin/env bash # pyreg.sh: registry-style store in pure bash. # # Here the directory tree IS the registry: keys are directories under...

  • Hot russian chick
     
  • scriptkidie teaches hacking
    (no subject) 22 messages   Wed, Oct 21, 2026 at 10:38 AM To: cjb@cjb.net, admin@d2jsp.org ipconfig google.com  i am the skr1p1k1dd13 i know ...
  • Holy motherfuck
      # !/ usr / bin / env python3 """pyreg: a tiny Windows-registry-style store that shows up as a Unix directory tree. Keys ...