#!/usr/bin/perl

# whohas, a Perl utility to display availability of source
# and binary packages from major Linux and BSD distributions
#
# Copyright (C) Philipp L. Wesche 2005-2011
#
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

use strict;
use sigtrap;

#TODO --fetch-unstable switch
#TODO make sure that debian's version numbers are from i386
#TODO get date info about debian, ubuntu (link is to changelog)
#TODO get size and date info about slackware packages from subsequent links  - postponed until slackware packages is online again
#TODO architecture tests for those that support several
#TODO Gentoo: only report two most recent for each package?
#TODO make more use of the conf directory, e.g. for Fedora, Sourcemage, so we download those indexes only sparingly, and save ourselves processing time; MAKE SURE YOU WRITE THE PROCESSED RESULTS FOR EASY PARSING ON THE NEXT RUN
#TODO we can also cache search results there to drastically reduce query time on subsequent queries
#TODO add option to override the cache (newly fetched file written to cache)
#TODO ubuntu: allow two releases: the long term supported and the most recent
#TODO allow searching on several packages; return results only for those distros that have a hit for each package, possibly in tabular format
#TODO deal gracefully with hyphens that may be present in some distros but not others, i.e. include extra hyphens in regexes, and allow user-specified hyphens to be absent
#TODO fix fedora, opensuse

use Env qw(HOME);
use threads;
use Getopt::Long;

my $confdir = "$HOME/.whohas";
# make .whohas directory in home directory
unless (-d $confdir) {
	mkdir ($confdir, 0755);
}

my @columns = (11,38,18,4,10,25);
my $cols = 6;

my $fedora_release		 =  16			;
my $ubuntu_current_release	 = "oneiric"		;
my $opensuse_major		 = "12"			;
my $opensuse_minor		 = "1"			;
my $mandrivaVersion		 = "2011.0"		;
my $openbsd_release		 = "5.0"		;
my $slackware_version_for_lp_net = "13.1"		;
my $slackfind_version		 = 9			; # corresponds to 13.1

my @distrosAvailable = ('arch','cygwin','debian','fedora','fink','freebsd','gentoo','macports','mandriva','netbsd','openbsd','opensuse','slack','sourcemage','ubuntu');
my %distrosSelected;
foreach (@distrosAvailable) {
	$distrosSelected{$_} = 1;
}

my @thrs;
my $here = 0;

my @distroSelections;
my $nothreads;
my $shallow;
GetOptions(
	"d=s" => \@distroSelections,
	"no-threads" => \$nothreads,
	"shallow" => \$shallow
);

if (@ARGV > 1) {
	die "Error:\tToo many parameters. Usage: $0 [--no-threads] [--shallow] [-d Dist1[,Dist2[,Dist3...]]] pkgname\n";
} elsif (@ARGV < 1) {
	die "Error:\tPlease specify a search term.\n";
}

#
# BUILD %distrosSelected
#

if (@distroSelections) {
	foreach (@distrosAvailable) {
		$distrosSelected{$_} = 0;
	}
	@distroSelections = split(/,/,join(',',@distroSelections));
	for my $distro (@distroSelections) {
		$distro =~ tr/A-Z/a-z/;
		if (		$distro =~ /archlinux/i) {	$distrosSelected{arch}	= 1;
		} elsif (	$distro =~ /slackware/i) {	$distrosSelected{slack}	= 1;
		} else {
			if (exists $distrosSelected{$distro}) {	# NB only due to previous setting of hash values for all known distros can we use this test to see if the distro is known
				$distrosSelected{$distro} = 1;
			} else {
				die "Unsupported distribution '$distro'\n";
			}
		}
	}
}

#
# DISPATCH TO SUBROUTINES, THREADED OR UNTHREADED
#

if ($ARGV[0] eq "whohasme") {
	print "Congratulations. You discovered an Easter egg. Maybe you can send a quick email to phi1ipp\@yahoo.com to say hello and tell the developer what you think of the software.\n";
	exit;
}

if (!$nothreads) {
	foreach (keys %distrosSelected) {
		if ($distrosSelected{$_}) {
			if ($_ eq 'arch') {
				$thrs[$here++] = threads->new(\&arch,		$ARGV[0]);
				$thrs[$here++] = threads->new(\&aur,		$ARGV[0]);
			} elsif ($_ eq 'slack') {
				$thrs[$here++] = threads->new(\&slack,		$ARGV[0]);
				$thrs[$here++] = threads->new(\&lp_net,		$ARGV[0]);
			} else {	# NB this is only safe because we've previously checked for illegal subs
				no strict "refs";
				$thrs[$here++] = threads->new(\&$_,		$ARGV[0]);
			}
		}
	}
	foreach (@thrs) {
		$_->join;
	}
} else {
	foreach (keys %distrosSelected) {
		if ($distrosSelected{$_}) {
			if ($_ eq 'arch') {
				&arch(		$ARGV[0]);
				&aur(		$ARGV[0]);
			} elsif ($_ eq 'slack') {
				&slack(		$ARGV[0]);
				&lp_net(	$ARGV[0]);
			} else {	# NB this is only safe because we've previously checked for illegal subs
				no strict "refs";
				&$_(		$ARGV[0]);
			}
		}
	}
}

sub fedora {
	my $baseurl = "http://download.fedora.redhat.com/pub/fedora/linux/releases/";
	my $distroname = "Fedora";
	my $arch = "i386";
	my $searchy = $_[0];
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	for (my $i = 16; $i <= $fedora_release; $i++) { # just the one release for now
#		my @fed_repos = ("core","Everything");
		my @fed_urls = ("$i/Everything/$arch/os/Packages/");
		my $file = "$confdir/$distroname\_$i.list";
		# if the list file exists and is recent, use its contents, otherwise download and parse a fresh copy
		if (-s $file && `date +%Y-%m-%d` =~ (split / /, `ls -l $file`)[6]) {
			open IN, $file;
			chomp (my @lines = <IN>);
			for (my $i = 0; $i<@lines;$i++) {
				($names[$i],$versions[$i],$sizes[$i],$dates[$i]) = split /\t/, $lines[$i];
			}
			close IN;
		} else {
			for (my $a = 0; $a < @fed_urls; $a++) {
				my @lines = split /\n/, &fetchdoc($baseurl.$fed_urls[0]);
				for (my $li = 0; $li < @lines; $li++) {
					if ($lines[$li] =~ /\/icons\/unknown\.gif/) {
						my @parts = split /\<|\>|\"/, $lines[$li];
						my $prev_release = $fedora_release - 1;
						$parts[22] =~ s/\.fc($fedora_release|$prev_release).+//;
						my ($name,$version) = &combos($parts[22]);
						push @dates, &month_to_digits($parts[32]);
						$parts[38] =~ s/^\s+//;
						push @sizes, $parts[38];
						push @names, $name;
						push @versions, $version;
						push @repos, "";
						push @urls, "";
					}
				}
			}
		}
		open OUT, ">$file";
		for (my $i = 0; $i < @urls;$i++) {
			print OUT "$names[$i]\t$versions[$i]\t$sizes[$i]\t$dates[$i]\n";
		}
		close OUT;
	}
	for (my $i = 0; $i < @names; $i++) {
		if ($names[$i] =~ /$searchy/i) {
			&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
		}
	}
	return ();
}

sub month_to_digits {
	$_[0] =~ s/JAN/01/i;
	$_[0] =~ s/FEB/02/i;
	$_[0] =~ s/MAR/03/i;
	$_[0] =~ s/APR/04/i;
	$_[0] =~ s/MAY/05/i;
	$_[0] =~ s/JUN/06/i;
	$_[0] =~ s/JUL/07/i;
	$_[0] =~ s/AUG/08/i;
	$_[0] =~ s/SEP/09/i;
	$_[0] =~ s/OCT/10/i;
	$_[0] =~ s/NOV/11/i;
	$_[0] =~ s/DEC/12/i;
	return ($_[0]);
}

sub lp_net {
	my $version = $slackware_version_for_lp_net;
	my @lines = split /\n/, &fetchdoc("http://www.linuxpackages.net/search_view.php?by=name&name=".$_[0]."&ver=".$version);
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /<tr bgcolor="#ffffff"><td>/) {
			push @names,    (split /">|<\/a/,   $lines[$i+1])[1];
			push @versions, (split /<\/td/,     $lines[$i+3])[0];
			push @repos,    (split /<\/td/,     $lines[$i+5])[0];
			my $url = (split /href="|">/, $lines[$i+8])[1];
			my @ret = &lp_net_details($url);
			push @urls, $url;
			push @sizes, $ret[0];
			push @dates, $ret[1];
			$i += 10;
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"lp.net",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub lp_net_details {
	my $size;
	my $date;
	my $findd = 0;
	my $finds = 0;
	my @lines = split /\n/, &fetchdoc($_[0]);
	for (my $i = 340; $i < @lines;$i++) {
		if ($lines[$i] =~ /Date :/) {
			if ($findd == 1) {
				die "encountered several dates for package\n";
			} else {
				$findd = 1;
			}
			$date = join "", (split //, (split /<b>|<\/b>/, $lines[$i])[1])[0..9];
		}
		if ($lines[$i] =~ /Filesize :/) {
			if ($finds == 1) {
				die "encountered several sizes for package\n";
			} else {
				$finds = 1;
			}
			my $presize = (split / /, (split /<b>|<\/b>/, $lines[$i])[1])[0];
			$size = &size_trim($presize*1000);
		}
	}
	return ($size,$date);
}

sub macports {
	my $baseurl = "http://www.macports.org";
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	my @lines = split /\n/, &fetchdoc($baseurl."/ports.php?by=name&substr=".$_[0]);
	for (my $i = 70; $i < @lines; $i++) {
		if ($lines[$i] =~ /<dt><b>/) {
			my @parties = split /\<dt\>\<b\>/, $lines[$i];
			for (my $javar = 1; $javar < @parties; $javar++) {
				my @parts = split /href="|">|<\/a><\/b> |<\/dt/, $parties[$javar];
				push @urls,     $parts[1];
				push @names,    $parts[2];
				push @versions, $parts[3];
				push @repos, "";
				push @sizes, "";
				push @dates, "";
			}
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"MacPorts",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}


sub fink {
	my $baseurl = "http://fink.sourceforge.net/pdb/";
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	my @lines = split /\n/, &fetchdoc($baseurl."browse.php?summary=".$_[0]);
	for (my $i = 60; $i < @lines; $i++) {
		if ($lines[$i] =~ /tr class=\"package\"/) {
			if ($lines[$i] =~ /^\<tr class\=\"pdbHeading\"\>/) {
				$lines[$i] =~ s/.*?\<\/tr\>//;
			}
			my @splitty = split /href\=\"|\"\>|\<\/a\>\<\/td\>\<td class=\"packageName\"\>|\<\/td\>\<td\>/, $lines[$i];
			push @urls, $splitty[3];
			push @names, $splitty[4];
			push @versions, $splitty[5];
			push @repos, "";
			push @sizes, "";
			push @dates, "";
		} elsif ($lines[$i] =~ /\<p\>Query took /) {
			last;
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Fink",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub fink_get_details {
	my @repos;
	my @versions;
	my @lines = split /\n/, &fetchdoc($_[0]);
	for (my $i = 60; $i < @lines; $i++) {
		if ($lines[$i] =~ /10\./) {
			my @parts = split /nowrap">|<\/div>/, $lines[$i];
			unless ($parts[5] =~ /not present/ or $parts[5] =~ /unsupported/) {
				if ($parts[1] =~ /<br>/) {
					push @repos, (split /<br>/, $parts[1])[0];
				} else {
					push @repos, $parts[1];
				}
				$parts[5] =~ s/\<\!\-\-.*\-\-\>//;
				push @versions, $parts[5];
			}
		}
		if ($lines[$i] eq '</table>') {
			last;
		}
	}
	return (\@versions,\@repos);
}

sub size_trim {
	# give at least two significant figures; if a 10^3 edge is encountered, put a dot
	my $leave =  length($_[0]) % 3;
	my $threes = (length($_[0]) - $leave) / 3;
	if ($leave == 0) {
		$leave = 3;
		$threes--;
	}
	
	my @parts = split //, $_[0];
	my $retval = join "", @parts[0..($leave-1)];
	if (length($retval)==1 && $threes > 0) {
		# add one more significant figure
		my $add_sf = $parts[$leave];
		if ($parts[$leave+1] > 4) {
			# rounding
			$add_sf++;
			if ($add_sf == 10) {
				$add_sf = 0;
				$retval++;
				if ($retval == 10) {
					return(&size_trim($retval*(1000**$threes)));
				}
			}
		}
		$retval .= ".$add_sf";
	} elsif (defined($parts[$leave]) && $parts[$leave] > 4) { # instead of defined(...), ($threes > 0) is also possible
		my $before = length($retval);
		# rounding
		$retval++;
		if (length($retval) > $before) {
			return(&size_trim($retval*(1000**$threes)));
		}
	}
	my @suffixes = ("k","M","G");
	if ($threes > 0) {
		$retval .= $suffixes[($threes-1)];
	}
	return $retval;
}


sub freebsd {
	my $query = "http://www.freebsd.org/cgi/ports.cgi?query=".$_[0]."&stype=all";
	my @lines = split /\n/, &fetchdoc($query);
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	my $now = 0;
	for (my $i = 50; $i<@lines; $i++) {
		if ($lines[$i] =~ /^<dt><b>/) {
			my @parts = split /"/, $lines[$i];
			($names[$now],$versions[$now]) = &combos_freebsd($parts[1]);
			my @subparts = split /\//, $parts[3];
			push @sizes, "";
			push @repos, $subparts[@subparts-2];
			push @urls,  "http://www.freebsd.org/cgi/pds.cgi?ports/".$subparts[@subparts-2]."/".$names[$now];
			$now++;
			push @dates, "";
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"FreeBSD",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}


sub sourcemage {
	my @grimoires = ("test","stable","binary","z-rejected","games");
	my @lines = split /\n/, &fetchdoc("http://codex.sourcemage.org/listing.txt");
	my @inirepos;
	my @ininames;
	my @iniversions;
	my @iniurls;
	my @inidates;
	my @inisizes;
	foreach (@lines) {
		my @comps = split /\^/, $_;
		for (my $a = 0; $a < @grimoires;$a++) {
			if (length($comps[$a+1]) > 0) {
				push @inirepos, $grimoires[$a];
				push @ininames, $comps[0];
				push @iniversions, $comps[$a+1];
				push @inisizes, "";
				push @iniurls,  "";
				push @inidates, "";
			}
		}
	}
	my ($p1,$p2,$p3,$p4,$p5,$p6) = &search_by_name(\@ininames,\@iniversions,\@inisizes,\@inidates,\@inirepos,\@iniurls,$_[0]);
	my @names    = @$p1;
	my @versions = @$p2;
	my @sizes    = @$p3;
	my @dates    = @$p4;
	my @repos    = @$p5;
	my @urls     = @$p6;
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Source Mage",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}


sub search_by_name { # versions, sizes, dates, repos, urls
	my ($p1,$p2,$p3,$p4,$p5,$p6,$search) = @_;
	my @ininames    = @$p1;
	my @iniversions = @$p2;
	my @inisizes    = @$p3;
	my @inidates    = @$p4;
	my @inirepos    = @$p5;
	my @iniurls     = @$p6;
	my @names;
	my @versions;
	my @sizes;
	my @dates;
	my @repos;
	my @urls;
	for (my $i = 0;$i<@ininames;$i++) {
		if ($ininames[$i] =~ /$search/i) {
			push @names,    $ininames[$i];
			push @repos,    $inirepos[$i];
			push @versions, $iniversions[$i];
			push @sizes,    $inisizes[$i];
			push @dates,    $inidates[$i];
			push @urls,     $iniurls[$i];
		}
	}
	return(\@names,\@versions,\@sizes,\@dates,\@repos,\@urls);
}

sub netbsd_old {
	my $netbsdbase = "ftp://ftp.netbsd.org/pub/NetBSD/packages/pkgsrc/";
	my @ininames;
	my @iniversions;
	my @iniurls;
	my @inirepos;
	my @inisizes;
	my @inidates;
	my $now = 0;
	my $distroname = "NetBSD";
	my $file = "$confdir/$distroname.list";
	# if the list file exists and is recent, use its contents, otherwise download and parse a fresh copy
	if (-s $file && `date +%Y-%m-%d` =~ (split / /, `ls -l $file`)[6]) {
		open IN, $file;
		chomp (my @lines = <IN>);
		for (my $i = 0; $i<@lines;$i++) {
			($ininames[$i],$iniversions[$i],$iniurls[$i]) = split /\t/, $lines[$i];
		}
		close IN;
	} else {
		my @lines = split /\n/, &fetchdoc($netbsdbase."README-all.html");
		for (my $i = 10; $i < @lines; $i++) {
			if ($lines[$i] =~ /^<!-- [0-9A-Za-z]/) {
				my @parts = split / /, $lines[$i];
				($ininames[$now],$iniversions[$now]) = &combos($parts[1]);
				$now++;
				@parts = split /a href="|">/, $lines[$i];
				push @iniurls, $netbsdbase.$parts[1];
				push @inirepos, "";
				push @inisizes, "";
				push @inidates, "";
			}
		}
		open OUT, ">$file";
		for (my $i = 0; $i < @iniurls;$i++) {
			print OUT "$ininames[$i]\t$iniversions[$i]\t$iniurls[$i]\n";
		}
		close OUT;
	}
	my ($p1,$p2,$p3,$p4,$p5,$p6) = &search_by_name(\@ininames,\@iniversions,\@inisizes,\@inidates,\@inirepos,\@iniurls,$_[0]);
	my @names    = @$p1;
	my @versions = @$p2;
	my @sizes    = @$p3;
	my @dates    = @$p4;
	my @repos    = @$p5;
	my @urls     = @$p6;
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub openbsd {
	if ($openbsd_release < 5) {
		&openbsd_older(@_);
	} else {
		&openbsd_newer(@_);
	}
	return();
}

sub openbsd_combos {
	my @parts = split /-/, $_[0];
	for (my $i = 1; $i < @parts; $i++) {
		if ($parts[$i] =~ /^[0-9]/) {
			return ( (join '-',@parts[0..($i-1)]), (join '-', @parts[$i..(@parts-1)]) );
		}
	}
}

sub openbsd_newer {
	my $rel = $openbsd_release;
	my $arch = "i386";
	my $baseurl = 'http://ftp.openbsd.org/pub/OpenBSD/'.$rel.'/packages/'.$arch.'/';
	my @names;
	my @versions;
	my @urls;
	my @repos;
	my @sizes;
	my @dates;
	my $distroname = "OpenBSD";
	my $file = "$confdir/$distroname\_$rel.list";
	# if the list file exists and is recent, use its contents, otherwise download and parse a fresh copy
	if (-s $file && `date +%Y-%m-%d` =~ (split / /, `ls -l $file`)[6]) {
		open IN, $file;
		chomp (my @lines = <IN>);
		for (my $i = 0; $i<@lines;$i++) {
			($names[$i],$versions[$i],$dates[$i],$sizes[$i]) = split /\t/, $lines[$i];
		}
		close IN;
	} else {
		my @lines = split /\n/, &fetchdoc($baseurl);
		my $now = 0;
		for (my $i = 0; $i < @lines; $i++) {
			if ($lines[$i] =~ /^<IMG SRC="\/icons\/compressed\.gif/) {
				my @firstParts = split /<A HREF="|\.tgz">|.tgz<\/A> +|  +/, $lines[$i];
				my $a = @names;
				($names[$a],$versions[$a]) = &openbsd_combos($firstParts[2]);
				push @dates, $firstParts[4];
				push @sizes, $firstParts[5];
			}
		}
		open OUT, ">$file";
		for (my $i = 0; $i < @urls;$i++) {
			print OUT "$names[$i]\t$versions[$i]\t$dates[$i]\t$sizes[$i]\n";
		}
		close OUT;
	}
	my $matcher = $_[0];
	for (my $i = 0; $i < @names; $i++) {
		if ($names[$i] =~ /$matcher/i) {
			&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
		}
	}
	return ();
}

sub openbsd_older {
	my $rel = $openbsd_release;
	my $arch = "i386";
	my $baseurl = "http://www.openbsd.org/".$rel.'_packages/';
	my @names;
	my @versions;
	my @urls;
	my @repos;
	my @sizes;
	my @dates;
	my $distroname = "OpenBSD";
	my $file = "$confdir/$distroname\_$rel.list";
	# if the list file exists and is recent, use its contents, otherwise download and parse a fresh copy
	if (-s $file && `date +%Y-%m-%d` =~ (split / /, `ls -l $file`)[6]) {
		open IN, $file;
		chomp (my @lines = <IN>);
		for (my $i = 0; $i<@lines;$i++) {
			($names[$i],$versions[$i],$urls[$i]) = split /\t/, $lines[$i];
		}
		close IN;
	} else {
		my @lines = split /\n/, &fetchdoc($baseurl.$arch.".html");
		my $now = 0;
		for (my $i = 0; $i < @lines; $i++) {
			if ($lines[$i] =~ /^<td><b><a/) {
				my @parts = split />|href=|\.tgz</, $lines[$i];
				push @urls, $baseurl.$parts[3];
				($names[$now],$versions[$now]) = &combos($parts[4]);
				$now++;
				push @repos, "";
				push @sizes, "";
				push @dates, "";
			}
		}
		open OUT, ">$file";
		for (my $i = 0; $i < @urls;$i++) {
			print OUT "$names[$i]\t$versions[$i]\t$urls[$i]\n";
		}
		close OUT;
	}
	my $matcher = $_[0];
	for (my $i = 0; $i < @repos; $i++) {
		if ($names[$i] =~ /$matcher/i) {
			&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
		}
	}
	return ();
}

sub cygwin {
	my $baseurl = "http://www.cygwin.com/packages/";
	my @names;
	my @versions;
	my @urls;
	my @repos;
	my @sizes;
	my @dates;
	my $distroname = "Cygwin";
	my $searchy = $_[0];
	my @lines = split /\n/, &fetchdoc($baseurl);
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /$searchy/i && $lines[$i] =~ /ware\.org\/icons\/ba/) {
			my $temp = (split /\"\>|\<\/a\>/, $lines[$i])[3];
			my @detailLines = split /\n/, &fetchdoc($baseurl.$temp);
			my $highest = '';
			for (my $a = 0; $a < @detailLines; $a++) { # incrementing ensures that the highest version number will prevail
								   # (higher ones occur lower down at time of writing)
				if ($detailLines[$a] =~ /<li><a href="/ && $detailLines[$a] !~ /\-src\<\/a\>\<\/li\>/) {
					$highest = (split /<a href="|">/, $detailLines[$a])[1];
				}
			}
			push @versions, (&combos($highest))[1];
			push @names,    $temp;
			push @dates,    '';
			push @urls,     '';
			push @sizes,    '';
			push @repos,    '';
		}
	}
	for (my $i = 0; $i < @names; $i++) {
		&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub gentoo {
	my $gentoobase = "http://gentoo-portage.com";
	my $distroname = "Gentoo";
	my @names;
	my @versions;
	my @urls;
	my @dates;
	my @lines = split /\n/, &fetchdoc($gentoobase."/Search?search=".$_[0]);
	my $name;
	my @repos;
	my @sizes;
	my @groups;
	for (my $i = 0; $i < @lines; $i++) { # starting value is a speed compromise
		if ($lines[$i] =~ /<div id\=\"search_results\"\>/) {
			for (my $a = $i+1; $a < @lines; $a++) {
				if ($lines[$a] =~ /\<\/div\>/) {
					if ($lines[$a] !~ /\<div\>/) {
						last;
					} else {
						my @parts = split /\<div\>/, $lines[$a];
						my @dosparts = split /\//, $parts[1];
						$dosparts[1] =~ / +$/;
						my $tempurl = $gentoobase."/".$dosparts[0]."/".$dosparts[1];
						my @newlines = split /\n/, &fetchdoc($tempurl);
						for (my $li = 0; $li < @newlines; $li++) {
							if ($newlines[$li] =~ /\<li class\=\"[a-z]+ebuildrow\"/) {
								my @tempbreak = split /\<b\>|\<\/b\>|-/, $newlines[$li+2];
								my $vernum;
								for (my $incrementa = 2; $incrementa < @tempbreak; $incrementa++) {
									if ($tempbreak[$incrementa] =~ /^[0-9]/) {
										$vernum = join "-", @tempbreak[$incrementa..(@tempbreak-2)];
									}									
								}
								push @names, $dosparts[1];
								push @groups, $dosparts[0];
								push @urls, $tempurl;
								push @versions, $vernum;
								push @repos, "";
								push @sizes, "";
								push @dates, "";
							}
						}
					}
				}
			}
			last;
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

# this almost works, will make the whole thing a lot faster!
#sub combos {
#	my @parts = split /-/, $_[0];
#	my $name;
#	my $version;
#	for (my $i = 1; $i < @parts-1; $i++) {
#		if ($parts[$i] =~ /^[0-9]/) {
#			$name = join "-", @parts[0..($i-1)];
#			$version = join "-", @parts[$i..(@parts-1)];
#			last;
#		}
#	}
#	return($name,$version);
#}

sub combos {
        my @chars = split //, $_[0];
        my $name;
        my $version; 
        for (my $i = 0; $i < @chars-1; $i++) {
                if ($chars[$i] eq "-"#) {
#			if (
&& $chars[$i+1] =~ /[0-9]/) {
                        	$name = join "", @chars[0..($i-1)];
                        	$version = join "", @chars[($i+1)..(@chars-1)];
                        	last;
#			} else {
#				$i++; #minor speed-up
#			}
                }
        }
        return($name,$version);
}

sub combor {
	my @chars = split //, $_[0];
	my $name;
	my $version;
	for (my $i = @chars - 1; $i >= 0; $i--) {
		if ($chars[$i] !~ /[0-9\-\.]/ && !($chars[$i] eq "i" && $chars[$i-1] eq "-" && $chars[$i+1] =~ /[6543]/)) {
			$name = join "", @chars[0..($i)];
			$version = join "", @chars[($i+2)..(@chars-1)];
			last;
		}
	}
	return($name,$version);
}

sub combos_freebsd {
	my @parts = split /-/, $_[0];
	my $name;
	my $version;
	for (my $i = 1; $i < @parts; $i++) {
		if ($parts[$i] =~ /^[0-9]/) {
			$name = join "-", @parts[0..($i-1)];
			$version = join "-", @parts[$i..(@parts-1)];
		}
	}
	return($name,$version);
}

sub slack {
	my $baseurl = 'http://slackfind.net';
	my @repos;
	my @groups;
	my @names;
	my @versions;
	my @urls;
	my @combos;
	my @sizes;
	my @dates;
	my @lines = split /\n/, &fetchdoc($baseurl."/en/packages/search/?name=".$_[0]."&distversion=".$slackfind_version);
	my $pages = 0;
	my @pagedURLs;

	# finding additional pages
	if (@lines > 800) {
		for (my $i = 800; $i < @lines; $i++) {
			if ($lines[$i] =~ /<span class="paging-button">/) {
				$pages++;
				if ($pages > 1) {
					my $tmp = (split /"/, $lines[$i])[3];
					$tmp =~ s/\&amp;/&/g;
					push @pagedURLs, $tmp;
				}
			}
		}
	}

	# finding content
	my $a = -1;
	for (; $a < @pagedURLs; $a++) {
		unless ($a < 0) {
			@lines = split /\n/, &fetchdoc($baseurl."/en/packages/search/".$pagedURLs[$a]);
		}
		for (my $i = 0; $i < @lines; $i++) {
			if ($lines[$i] =~ /result-package-name/) {
				my $a = @repos;
				($names[$a],$versions[$a]) = &slack_combos($lines[$i+1]);
			} elsif ($lines[$i] =~ /result-repository/) {
				my $tmp = $lines[$i+1];
				push @repos, (split /<span>| <a href/, $tmp)[1];
			}
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Slackware",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub slack_combos {
	$_[0] =~ s/^\s*//;
	my @parts = split /-/, $_[0];
	return ((join '-', @parts[0..(@parts-4)]),(join '-', @parts[(@parts-3)..(@parts-1)]));
}

sub slack_old {
	my $slackbase  = "http://packages.slackware.it/";
	my @repos;
	my @groups;
	my @names;
	my @versions;
	my @urls;
	my @combos;
	my @sizes;
	my @dates;
	my @lines = split /\n|<br>|<\/td>/, &fetchdoc($slackbase."/search.php?v=current&t=1&q=".$_[0]);
	my $now = 0;
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /class="pkgtitle"/) {
			my @parts = split /a href="|">|<\/a> in |<\/div>/, $lines[$i];
			$parts[4] =~ s/slackware\///;
			push @groups, $parts[4];
			push @repos, "";
			($names[$now],$versions[$now]) = &combor($parts[3]);
			$now++;
			push @urls, $slackbase.$parts[2];
			push @dates, "";
			push @sizes, "";
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Slackware",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub debian_sizes {
	my @lines = split /\n/, &fetchdoc($_[0]);
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /download\">i386<\/a/) {
			my @newparts = split /\"size\"\>|<\/td>/, $lines[$i+3];
			return &debian_size_convert($newparts[1]);
		}
	}
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /download\">all<\/a/) {
			my @newparts = split /\"size\"\>|<\/td>/, $lines[$i+3];
			return &debian_size_convert($newparts[1]);
		}
	}
}

sub debian_size_convert {
	if ($_[0] =~ s/\&nbsp;kB$//) {
		$_[0] =~ s/,//g;
		my @parts = split /\./, $_[0];
		my @partses = split //, $parts[1];
		if ($partses[0] >= 5) {
			$parts[0]++;
		}
		if ($parts[0] < 1000) {
			return $parts[0]."K";
		} else {
			my $val = round($parts[0]/1024);
			return $val."M";
		}
	} else {
		die "Strange packet size encountered: $_[0]\n";
	}
}

sub debian {
	my @dists = ( 'stable', 'testing' );
	&debuntu('http://packages.debian.org','Debian',\@dists,$_[0]);
	return();
}

sub ubuntu {
	my @array = ($ubuntu_current_release);
	&debuntu('http://packages.ubuntu.com','Ubuntu',\@array,$_[0]);
	return();
}

sub debuntu {
	my ($baseurl,$distname,$releaseArrayP,$searchTerm) = @_;
	my @dists = @$releaseArrayP;
	my @names;
	my @repos;
	my @groups;
	my @versions;
	my @urls;
	my @sizes;
	my @dates;
	for (my $x = 0; $x < @dists; $x++) {
		my @lines = split /\n/, &fetchdoc($baseurl."/search?keywords=".$searchTerm."&searchon=names&suite=".$dists[$x]."&section=all");
		for (my $i = 50; $i < @lines; $i++) {
			if ($lines[$i] =~ /<h3>Package /) {
				my $name = (split /h3>Package |<\/h3>/, $lines[$i])[1];
				push @names, $name;
				my @parts = split /href\=\"|\"\>|<\/a\>/, $lines[$i+3];
				$parts[4] =~ s/ \(|\)://g;
				push @groups, $parts[4];
				push @repos, $dists[$x];
				push @urls,  $baseurl.$parts[2];
				push @dates, "";
				@parts = split />|: /, $lines[$i+6];
				$parts[1] =~ s/ \[\<strong.*//;
				push @versions, $parts[1];
				$i += 11;
			}
		}
	}
	unless ($shallow) {
		if (!$nothreads) {
			my @thr;
			for (my $i = 0; $i < @urls; $i++) {
				push @thr, threads->new(\&debian_sizes, $urls[$i]);
			}
			for (my $i = 0; $i < @thr; $i++) {
				push @sizes, $thr[$i]->join;
			}
		} else {
			for (my $i = 0; $i < @urls; $i++) {
				push @sizes, &debian_sizes($urls[$i]);	# TODO but we want installed size - or both?
			}
		}
	} else {
		for (my $i = 0; $i < @urls; $i++) {
			push @sizes, "";
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,$distname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub aur {
	my $aurbase    = "http://aur.archlinux.org";
	my $stop;
	my @lines = split /\n/, &fetchdoc($aurbase."/packages.php?PP=1000&K=".$_[0]); # 1000 should be enough

	my @repos;
	my @names;
	my @versions;
	my @combos;
	my @dates;
	my @urls;
	my @sizes;
	my @dates;
	my $indicator = 0;
	for (my $i = 100; $i < @lines; $i++) {  # 100 is a compromise between safety and efficiency
		if ($lines[$i] =~ /<td class='data/) {
			$indicator++;
			if ($indicator == 1) {
				push @repos,    &arch_site_get_cont($lines[$i]);
				push @sizes, "";
				push @dates, "";
			} elsif ($indicator == 2) {
				push @combos,    &arch_site_get_cont($lines[$i]);
				push @urls,      $aurbase."/".&aur_site_get_url ($lines[$i]);
			} elsif ($indicator == 6) {
				$indicator = 0;
			}
			# 2 is package group, 4 is votes, 5 is description, 6 is maintainer
		}
	}
	foreach (@combos) {
		my @units = split / /, $_;
		push @names,    $units[0];
		push @versions, $units[1];
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Arch",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub arch {
	my $archbase   = "http://www.archlinux.org";
	my @lines = split /\n/, &fetchdoc($archbase."/packages/?arch=i686&repo=&q=".$_[0]."&last_update=&limit=all");

	my @repos;
	my @names;
	my @versions;
	my @dates;
	my @urls;
	my @sizes;
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /\<tr class\=\"pkgr/) {
			for (my $a = 2; $a < 11;$a++) {
				my $temp = $lines[$a+$i];
				$temp =~ s/.*\<td\>|\<\/td\>//g;
				if ($a == 3) {
					push @repos,$temp;
				} elsif ($a == 4) {
					push @urls, $archbase.&arch_site_get_url ($lines[$i+$a]);
					push @names, &arch_site_ger_cont($lines[$i+$a]);
				} elsif ($a == 6) {
					$temp =~ s/<span style=\".*\">|<\/span>//g;
					push @versions, $temp;
				} elsif ($a == 10) {
					push @dates,    $temp;
				}
			}
			# 2 is package group, 5 is description
			push @sizes, "";
			$i += 7;
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Arch",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub arch_site_get_url {
	my $temp = $_[0];
	$temp =~ s/.*\<a href=\"|\">.*//g;
	return $temp;
}

sub arch_site_get_cont {
 	my @parts = split />/, $_[0];
 	for (my $i = 0; $i < @parts; $i++) {
 		if ($parts[$i] =~ /^[A-Za-z0-9]/) {
 			return (split /</, $parts[$i])[0];
 		}
 	}
}

sub arch_site_ger_cont {
	my $temp = $_[0];
	$temp =~ s/.*\"\>|\<\/a.*//g;
	return $temp;
}

sub aur_site_get_url {
	return (split /\'/, $_[0])[5];
}

sub fetchdoc {
	use LWP::UserAgent;

	my $url = $_[0];
	my $silent = 0;
	if (@_ == 2 && $_[1] eq "silent") {
		$silent = 1;
	}

	$url =~ s/\&amp\;/\&/ig;   # convert &amp; to &

	my $ua = LWP::UserAgent->new;
	$ua->env_proxy;
        my @firstline;
        my @response;
        for (my $count = 0; ; ++$count) {   # termination condition inside loop
                my $req = HTTP::Request->new(GET => $url);
                my $res = $ua->request($req)->as_string;
                @response = split (/\n/, $res);
                @firstline = split (/ /, $response[0]);
		my $restest = 0;
		if (@firstline == 3) {
			$restest = $firstline[1];
		} elsif (@firstline > 3) {
			$restest = $firstline[0];
		}
		if ($restest == 200 || $response[0] =~ /200 OK/) { #NB the matching expression added specifically for NetBSD package page!
		# server response 200 is a stringent criterion, but should work
			last;
		} elsif ($count > 4) {   # loop termination condition
			unless ($silent == 1) {
				print ("Tried fetching \"$url\" five times. Giving up.\n");
			}
			return ();
			last;
		}
	}
    my $end = @response - 1;
    my $finaldoc = join ("\n", @response[14..$end]);
    return ($finaldoc);
}

sub pretty_print {
	my $n = $_[0];
	my @colwidths = @_[1..$n];
	my @colvals = @_[($n+1)..(@_-1)];
	for (my $i = 0; $i < @colwidths;$i++) {
		if (length($colvals[$i]) > $colwidths[$i]) {
			my @letters = split //, $colvals[$i];
			print join "", @letters[0..($colwidths[$i]-1)];
			print " ";
		} else {
			print $colvals[$i];
			for (my $a = 0; $a < $colwidths[$i] + 1 - length($colvals[$i]); $a++) {
				print " ";
			}
		}
	}
	print $colvals[@colvals-1]."\n"; #last column is unrestricted in length
}

sub prep_suse_repo {
	if ($_[0] =~ /http\:\/\/download\.opensuse\.org/) {
		if ($_[0] =~ /repositories/) {
			if ($_[0] =~ /home\:/) {
				$_[0] =~ s/^http\:\/\/download\.opensuse\.org\/repositories\/home\:\///;
				$_[0] =~ s/\/openSUSE_11.0//;
				$_[0] =~ s/\:\/.*$//g;
				return $_[0];
			} elsif ($_[0] =~ /http\:\/\/.*:\//) {
				$_[0] =~ s/http\:\/\/download\.opensuse\.org\/repositories\///;
				$_[0] =~ s/\:\/.*//;
				return $_[0];
			} else {
				$_[0] =~ s/http\:\/\/download\.opensuse\.org\/repositories\///;
				$_[0] =~ s/\/.*//;
				return $_[0];
			}
		} elsif ($_[0] =~ /http\:\/\/download\.opensuse\.org\/distribution\// && $_[0] =~ /repo\/oss/) {
			return "suse/oss";
		}
	} elsif ($_[0] =~ "http://packman.iu-bremen.de/suse/") {
		return "packman";
	} else {
		return $_[0];
	}
}

sub opensuse {
	my $release_string = $opensuse_major.'.'.$opensuse_minor;
	my $major = $opensuse_major;
	my $minor = $opensuse_minor;
	my $opensusebase = "http://software.opensuse.org";
	my @names;
	my @repos;
	my @groups;
	my @versions;
	my @urls;
	my @sizes;
	my @dates;
	my $distroname = "openSUSE";

	my @lines = split /\n/, &fetchdoc($opensusebase."/search?q=".$_[0]."&baseproject=".$distroname."%3A".$release_string."&lang=en&exclude_debug=true");
	my $continuity = 2;
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /div class="search_o_title"/) {
			push @names, (split /<\/?h3>/, $lines[($i+2)])[1];
			for (my $a = $i+1; $a < $i + 20; $a++) {
				if ($lines[$a] =~ /<h4><a class="blue_over" href=/) {
					my @temps = split /\"\>|\<\/a\>/, $lines[$a];
					$temps[1] =~ s/\/openSUSE_$major\.$minor$//;
					$temps[1] =~ s/^openSUSE\:$major\.$minor\///;
					$temps[1] =~ s/\/$major\.$minor$//;
					$temps[1] =~ s/^home/\~/;
					push @repos, $temps[1];
					last;
				}
			}
			for (my $a = $i + 10; $a < $i + 50; $a++) {
				if ($lines[$a] =~ /a style="margin-right: 2em"  href/) {
					my @parts = split /\"\>|\<\/a\>/, $lines[$a];
					my @segments = split /\-/, $parts[1];
					push @versions, $segments[(@segments-2)];
					last;
				}
			}
			push @urls,   '';
			push @sizes,  '';
			push @dates,  '';
			push @groups, '';
		}
		if ($lines[$i] =~ /type:'post', url:'\/search\/search.*'\}\); return false;">$continuity<\/a>/) {
			push @lines,  split /\n/, &fetchdoc($opensusebase."?q=".$_[0]."&baseproject=openSUSE:".$release_string.'&p='.$continuity);
			$continuity++;
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return();
}

sub netbsd_pkgsrc_size {
	my @retvals;
	my $continueAt = 0;
	my @lines = split /\n|<br\/>/, &fetchdoc($_[0]);
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /Filesize:/) {
			my @parts = split /<\/b> /, $lines[$i];
			push @retvals, &sizeconvert($parts[1]);
			$continueAt = $i;
			last;
		}
	}
	if ($continueAt == 0) {
		push @retvals, "";
	}
	$retvals[1] = ""; #just in case there's no match found
	for (my $i = $continueAt; $i < @lines; $i++) {
		if ($lines[$i] =~ /Updated to version|Package added to/) {
			my @parts = split /<b>|<\/b>/, $lines[$i];
			$retvals[1] = $parts[1];
			last;
		}
	}
	return @retvals;
}

sub round {
    my($number) = shift;
    return int($number + .5);
}

sub sizeconvert {
	if ($_[0] =~ s/ KB$//) {
		my @parts = split /\./, $_[0];
		my @partses = split //, $parts[1];
		if ($partses[0] >= 5) {
			$parts[0]++;
		}
		if ($parts[0] < 1000) {
			return $parts[0]."K";
		} else {
			my $val = round($parts[0]/1024);
			return $val."M";
		}
	} else {
		die "Strange packet size encountered: $_[0]\n";
	}
}

sub netbsd {
	my @lines = split /\n|<br\/>/, &fetchdoc("http://pkgsrc.se/search.php?so=".$_[0]);
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	for (my $i = 0; $i < @lines; $i++) {
		if ($lines[$i] =~ /version.+maintainer/) {
			my @parts = split /href="|<\/a>, <em><b>version |<\/b>, maintainer|\">/, $lines[$i];
			push @urls, 	$parts[1];
			push @versions, $parts[3];
			my @subparts = split /\//, $parts[2];
			push @repos, $subparts[0];
			push @names, $subparts[1];
		}
	}
	unless ($shallow) {
		if (!$nothreads) {
			my @thr;
			for (my $i = 0; $i < @urls; $i++) {
				push @thr, threads->new(\&netbsd_pkgsrc_size, $urls[$i]);
			}
			for (my $i = 0; $i < @thr; $i++) {
				($sizes[$i],$dates[$i]) = $thr[$i]->join;
			}
		} else {
			for (my $i = 0; $i < @urls; $i++) {
				($sizes[$i],$dates[$i]) = &netbsd_pkgsrc_size($urls[$i]);
			}
		}
	} else {
		for (my $i = 0; $i < @urls; $i++) {
			($sizes[$i],$dates[$i]) = ("","");
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"NetBSD",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return();
}

sub fedora_admin {
# THIS is a possibility, but offers no date or size info
	my $query = "https://admin.fedoraproject.org/pkgdb/search/package/?searchwords=".$_[0]."&operator=AND&release=19&searchon=name";
	my @lines = split /\n/, &fetchdoc($query);
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	my $now = 0;
	for (my $i = 0; $i<@lines; $i++) {
		if ($lines[$i] =~ /unique_tag/) {
			# typically, this is where most of the text processing goes:
			# getting the info and putting it in appropriate arrays

			# use subroutine "combos" if the name and version are represented as, firefox-1.0.6, with the hyphen, and the version number starting with a digit
			my $anchor = "something";
			($names[$now],$versions[$now]) = &combos($anchor);
			$now++;
			push @names,    "";
			push @versions, "";
			push @repos,    "";
			push @sizes,    "";
			push @urls,     "";
			push @dates,    "";
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Distroname",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub template_query {
	my $query = "url";
	my @lines = split /\n/, &fetchdoc($query);
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	my $now = 0;
	for (my $i = 0; $i<@lines; $i++) {
		if ($lines[$i] =~ /unique_tag/) {
			# typically, this is where most of the text processing goes:
			# getting the info and putting it in appropriate arrays

			# use subroutine "combos" if the name and version are represented as, firefox-1.0.6, with the hyphen, and the version number starting with a digit
			my $anchor = "something";
			($names[$now],$versions[$now]) = &combos($anchor);
			$now++;
			push @names,    "";
			push @versions, "";
			push @repos,    "";
			push @sizes,    "";
			push @urls,     "";
			push @dates,    "";
		}
	}
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,"Distroname",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub template_listing {
	my @ininames;
	my @iniversions;
	my @iniurls;
	my @inirepos;
	my @inisizes;
	my @inidates;
	my $now = 0;
	my $distroname = "mydistro";
	my $base = "url";
	# prepare a list file
	my $file = "$confdir/$distroname.list";
	# if the list file exists and is recent, use its contents, otherwise download and parse a fresh copy
	if (-s $file && `date +%Y-%m-%d` =~ (split / /, `ls -l $file`)[6]) {
		open IN, $file;
		chomp (my @lines = <IN>);
		for (my $i = 0; $i<@lines;$i++) {
			# get back any info that you put in the file
			($ininames[$i],$iniversions[$i],$iniurls[$i]) = split /\t/, $lines[$i];
		}
		close IN;
	} else {
		# download fresh copy
		my @lines = split /\n/, &fetchdoc($base."README-all.html");
		for (my $i = 0; $i < @lines; $i++) {
			if ($lines[$i] =~ /unique_tag/) {
				# extract all info from the downloaded list
				my @parts = split / /, $lines[$i];
				# use subroutine "combos" to separate name and version number
				($ininames[$now],$iniversions[$now]) = &combos($parts[1]);
				$now++;
				# any info you couldn't get, put a blank in
				push @iniurls,  "";
				push @inirepos, "";
				push @inisizes, "";
				push @inidates, "";
			}
		}
		open OUT, ">$file";
		for (my $i = 0; $i < @iniurls;$i++) {
			# store the available info in the file
			print OUT "$ininames[$i]\t$iniversions[$i]\t$iniurls[$i]\n";
		}
		close OUT;
	}
	# search by hand
	my ($p1,$p2,$p3,$p4,$p5,$p6) = &search_by_name(\@ininames,\@iniversions,\@inisizes,\@inidates,\@inirepos,\@iniurls,$_[0]);
	my @names    = @$p1;
	my @versions = @$p2;
	my @sizes    = @$p3;
	my @dates    = @$p4;
	my @repos    = @$p5;
	my @urls     = @$p6;
	for (my $i = 0; $i < @repos; $i++) {
		&pretty_print($cols,@columns,$distroname,$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}

sub mandriva_combos {
	$_[0] =~ s/\<\/a\>.*//;
	my @parts = split /-/, $_[0];
	for (my $i = 1; $i < @parts; $i++) {
		if ($parts[$i] =~ /^[0-9]/) {
			return ( (join '-',@parts[0..($i-1)]), (join '-', @parts[$i..(@parts-1)]) );
		}
	}
}

sub mandriva {
	my $baseurl = "http://sophie.zarb.org";
	my @names;
	my @versions;
	my @dates;
	my @sizes;
	my @repos;
	my @urls;
	# NB this server also supports exact matching
	my @lines = split /\n/, &fetchdoc($baseurl."/search?search=".$_[0]."&type=fuzzyname&deptype=P&distribution=Mandriva&release=".$mandrivaVersion."&arch=");
	for (my $i = 350; $i < @lines; $i++) {
		#TODO need to check for possible further pages (lists 20 per page)
		#TODO ajax or xml::rpc access might have advantages w.r.t. paging (i.e. none required) 
		if ($lines[$i] =~ /<div class="sophie_search_list">/) {
			my $a = @names;
			push @urls, (split /"/, $lines[$i+1])[1];
			($names[$a],$versions[$a]) = &mandriva_combos($lines[$i+2]); # name, version, arch
		}
	}
	for (my $i = 0; $i < @urls; $i++) {
		&pretty_print($cols,@columns,"Mandriva",$names[$i],$versions[$i],$sizes[$i],$dates[$i],$repos[$i],$urls[$i]);
	}
	return ();
}
