#!/pro/bin/perl

use strict;
use warnings;

use Data::Peek;
use WWW::Mechanize;
use Text::CSV_XS;
use HTML::TreeBuilder;

binmode STDERR, "utf8";

my $csv  = Text::CSV_XS->new ({
    binary	=> 1,
    eol		=> "\n",
    });
my $mech = WWW::Mechanize->new (
#   agent	=> "Opera/9.50",
    autocheck	=> 1,
    cookie_jar	=> {},
    );
$mech->agent_alias ("Windows Mozilla");

my %cat = (
    1 => "Kwalificaties",
    5 => "Experimentele opleiding",
    );
foreach my $cat (1 .. 5) {
    open my $fh, ">:utf8", "crb-$cat.csv";

    print STDERR "Starting from home\n";
    $mech->get ("http://www.cfi.nl/Public/CreboWeb/crebovraag.asp?menuOptie=1-$cat");
    print STDERR "Click on OCW\n";
    $mech->follow_link (text => "Beroepsonderwijs OCW");

    # Now click on [Zoeken]
    $mech->submit_form (form_number => 1);

    my $page = 1;
    NEWPAGE: while ($page) {
	my $tree = HTML::TreeBuilder->new;
	$tree->parse_content ($mech->content ());
	#print STDERR $tree->as_HTML (undef, "  ", {});

	foreach my $tr ($tree->look_down (_tag => "tr")) {
	    my @td = map { $_->as_text } $tr->look_down (_tag => "td");

	    if (@td == 2 && $td[0] =~ m/^[0-9]+$/) {
		$csv->print ($fh, \@td);
		print STDERR "## $td[0]\t$td[1]\n";
		next;
		}

	    print STDERR ">> @td\n";
	    if ("@td" =~ m/\s+volgende\s+>/) {
		$mech->follow_link (text => "volgende >");
		redo NEWPAGE;
		}
	    }
	last;
	}
    close $fh;
    }

__END__
    # "Aantal gevonden: 615" => 41 pages

    my $page = 1;
    # http://www.cfi.nl/Public/CreboWeb/raadplegen.asp?menuoptie=1-5&zoekwaarde=&pagina=23&actie=zoek&lijstvolgorde=1&straal=&postcode=
    while ($mech->get ("$base?menuoptie=1-$cat&pagina=$page&$actn")) {
	print STDERR "Fetched page $cat-$page ...\n";
	my $tree = HTML::TreeBuilder->new;
	$tree->parse_content ($mech->content ());
	open my $fh, ">:utf8", "crb-$cat-$page.html" or die "crb-$page.html: $!\n";
	print   $fh $tree->as_HTML (undef, "  ", {});
	close   $fh;
	$page++ > 10 and last;
	}
    }
