Initial Commit

This commit is contained in:
Riley Schneider
2025-12-03 16:38:10 +01:00
parent c5e26bf594
commit b732d8d4b5
17680 changed files with 5977495 additions and 2 deletions

1311
database/perl/vendor/lib/CPAN/Mini.pm vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,209 @@
use strict;
use warnings;
package CPAN::Mini::App;
$CPAN::Mini::App::VERSION = '1.111016';
# ABSTRACT: the guts of the minicpan command
#pod =head1 SYNOPSIS
#pod
#pod #!/usr/bin/perl
#pod use CPAN::Mini::App;
#pod CPAN::Mini::App->run;
#pod
#pod =cut
use CPAN::Mini;
use File::HomeDir;
use File::Spec;
use Getopt::Long qw(:config no_ignore_case);
use Pod::Usage 1.00;
sub _display_version {
my $class = shift;
no strict 'refs';
print "minicpan",
($class ne 'CPAN::Mini' ? ' (from CPAN::Mini)' : q{}),
", powered by $class ", $class->VERSION, "\n\n";
exit;
}
#pod =method run
#pod
#pod This method is called by F<minicpan> to do all the work. Don't rely on what it
#pod does just yet.
#pod
#pod =cut
sub _validate_log_level {
my ($class, $level) = @_;
return $level if $level =~ /\A(?:fatal|warn|debug|info)\z/;
die "unknown logging level: $level\n";
}
sub run {
my ($class) = @_;
my $minicpan = $class->initialize_minicpan;
$minicpan->update_mirror;
}
sub initialize_minicpan {
my ($class) = @_;
my $version;
my %commandline;
my @option_spec = $class->_option_spec();
GetOptions(\%commandline, @option_spec) or pod2usage(2);
# These two options will cause the program to exit before finishing ->run
pod2usage(1) if $commandline{help};
$version = 1 if $commandline{version};
# How noisy should we be?
my $debug = $commandline{debug};
my $log_level = $commandline{log_level};
my $quiet = $commandline{qq} ? 2 : $commandline{quiet};
die "can't mix --debug, --log-level, and --debug\n"
if defined($quiet) + defined($debug) + defined($log_level) > 1;
# Set log_level accordingly
$quiet ||= 0;
$log_level = $debug ? 'debug'
: $quiet == 1 ? 'warn'
: $quiet >= 2 ? 'fatal'
: $log_level ? $log_level
: undef;
my %config = CPAN::Mini->read_config({
log_level => 'info',
%commandline
});
$config{class} ||= 'CPAN::Mini';
# Override config with commandline options
%config = (%config, %commandline);
$config{log_level} = $log_level || $config{log_level} || 'info';
$class->_validate_log_level($config{log_level});
eval "require $config{class}";
die $@ if $@;
_display_version($config{class}) if $version;
if ($config{remote_from} && ! $config{remote}) {
$config{remote} = $config{class}->remote_from(
$config{remote_from},
$config{remote},
$config{quiet},
);
}
$config{remote} ||= 'http://www.cpan.org/';
pod2usage(2) unless $config{local} and $config{remote};
$|++;
# Convert dirmode string to a real octal value, if given
$config{dirmode} = oct $config{dirmode} if $config{dirmode};
# Turn the 'perl' option into 'skip_perl', for backward compatibility
$config{skip_perl} = not delete $config{perl};
return $config{class}->new(%config);
}
sub _option_spec {
return qw<
class|c=s
help|h
version|v
quiet|q+
qq
debug
log_level|log-level=s
local|l=s
remote|r=s
dirmode|d=s
offline
force|f
perl
exact_mirror|x
timeout|t=i
config_file|config|C=s
remote-from=s
>;
}
#pod =head1 SEE ALSO
#pod
#pod Randal Schwartz's original article, which can be found here:
#pod
#pod http://www.stonehenge.com/merlyn/LinuxMag/col42.html
#pod
#pod =cut
1;
__END__
=pod
=encoding UTF-8
=head1 NAME
CPAN::Mini::App - the guts of the minicpan command
=head1 VERSION
version 1.111016
=head1 SYNOPSIS
#!/usr/bin/perl
use CPAN::Mini::App;
CPAN::Mini::App->run;
=head1 METHODS
=head2 run
This method is called by F<minicpan> to do all the work. Don't rely on what it
does just yet.
=head1 SEE ALSO
Randal Schwartz's original article, which can be found here:
http://www.stonehenge.com/merlyn/LinuxMag/col42.html
=head1 AUTHORS
=over 4
=item *
Ricardo SIGNES <rjbs@cpan.org>
=item *
Randal Schwartz <merlyn@stonehenge.com>
=back
=head1 COPYRIGHT AND LICENSE
This software is copyright (c) 2004 by Ricardo SIGNES.
This is free software; you can redistribute it and/or modify it under
the same terms as the Perl 5 programming language system itself.
=cut

View File

@@ -0,0 +1,21 @@
package CPAN::Mini::Portable;
use 5.008;
use strict;
use warnings;
use Portable ();
use CPAN::Mini 0.575 ();
our $VERSION = '1.23';
our @ISA = 'CPAN::Mini';
sub new {
# Use the portable values as defaults,
# completely ignoring any passed params
my $minicpan = Portable->default->minicpan;
# Hand off to the parent class
return $_[0]->SUPER::new( %$minicpan );
}
1;

351
database/perl/vendor/lib/CPAN/SQLite.pm vendored Normal file
View File

@@ -0,0 +1,351 @@
# $Id: SQLite.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use File::HomeDir;
require File::Spec;
use Cwd;
require CPAN::SQLite::META;
# an array ref of distributions to ignore indexing
my $ignore = [qw(SpreadSheet-WriteExcel-WebPivot)];
our $db_name = 'cpandb.sql';
use constant WIN32 => $^O eq 'MSWin32';
sub new {
my $class = shift;
my %args = @_;
my ($CPAN, $update_indices);
my $db_dir = $args{db_dir};
my $urllist = [];
my $keep_source_where;
# for testing under Darwin, must load CPAN::MyConfig contained
# in PERL5LIB, as File::HomeDir doesn't use this
if ($ENV{CPAN_SQLITE_TESTING}) {
eval { require CPAN::MyConfig; };
}
eval { require CPAN; CPAN::HandleConfig->load; };
if (not $@ and not defined $args{CPAN}) {
$CPAN = $CPAN::Config->{cpan_home};
$db_dir = $CPAN;
$keep_source_where = $CPAN::Config->{keep_source_where};
$urllist = $CPAN::Config->{urllist};
# Sometimes this directory doesn't exist (like on new installations)
unless (-d $CPAN) {
eval { File::Path::mkpath($CPAN); }; # copied from CPAN.pm
}
die qq{The '$CPAN' directory doesn't exist} unless -d $CPAN;
$update_indices = 0;
} else {
$CPAN = $args{CPAN} || '';
die qq{Please specify the CPAN location} unless defined $CPAN;
die qq{The '$CPAN' directory doesn't exist} unless (-d $CPAN);
$update_indices = (-f File::Spec->catfile($CPAN, 'MIRRORING.FROM')) ? 0 : 1;
}
push @$urllist, q{http://www.cpan.org/};
$db_dir ||= cwd;
my $self = {
%args,
CPAN => $CPAN,
update_indices => $update_indices,
db_name => $db_name,
urllist => $urllist,
keep_source_where => $keep_source_where,
db_dir => $db_dir
};
return bless $self, $class;
}
sub index {
my ($self, %args) = @_;
require CPAN::SQLite::Index;
my %wanted = map { $_ => $self->{$_} } qw(CPAN ignore update_indices db_name db_dir keep_source_where setup reindex urllist);
my $log_dir = $self->{CPAN} || $self->{db_dir};
die qq{Please create the directory '$log_dir' first} unless -d $log_dir;
my $index = CPAN::SQLite::Index->new(%wanted, %args, log_dir => $log_dir);
$index->index() or do {
warn qq{Indexing failed!};
return;
};
return 1;
}
sub query {
my ($self, %args) = @_;
require CPAN::SQLite::Search;
my %wanted = map { $_ => $self->{$_} } qw(max_results CPAN db_name db_dir meta_obj);
my $query = CPAN::SQLite::Search->new(%wanted, %args);
%wanted = map { $_ => $self->{$_} } qw(mode query id name);
$query->query(%wanted, %args) or do {
warn qq{Query failed!};
return;
};
my $results = $query->{results};
return unless defined $results;
$self->{results} = $query->{results};
return 1;
}
1;
=head1 NAME
CPAN::SQLite - maintain and search a minimal CPAN database
=head1 VERSION
version 0.219
=head1 SYNOPSIS
my $obj = CPAN::SQLite->new(CPAN => '/path/to/CPAN');
$obj->index(setup => 1);
$obj->query(mode => 'dist', name => 'CPAN');
my $results = $obj->{results};
=head1 DESCRIPTION
This package is used for setting up, maintaining, and
searching a CPAN database consisting of the information
stored in the two main CPAN indices:
F<$CPAN/modules/02packages.details.txt.gz> and
F<$CPAN/authors/01mailrc.txt.gz>. It should be
considered at an alpha stage of development.
One begins by creating the object as
my $obj = CPAN::SQLite->new(%args);
which accepts the following arguments:
=over 3
=item * C<CPAN =E<gt> '/path/to/CPAN'>
This specifies the path to where the index files are
to be stored. This could be a local CPAN mirror,
defined here by the presence of a F<MIRRORED.BY> file beneath
this directory, or a local directory in which to store
these files from a remote CPAN mirror. In the latter case,
the index files are fetched from a remote CPAN mirror,
using the same list that C<CPAN.pm> uses, if this is
configured, and are updated if they are more than one
day old.
If the C<CPAN> option is not given, it will default
to C<cpan_home> of L<CPAN>, if this is configured,
with the index files found under C<keep_source_where>.
A fatal error results if such a directory isn't found.
Updates to these index files are assumed here to be
handled by C<CPAN.pm>.
=item * C<db_dir =E<gt> '/path/to/db/dir'>
This specifies the path to where the database file is
found. If not given, it defaults to the
C<cpan_home> directory of C<CPAN.pm>, if present, or to
the directory in which the script was invoked. The name
of the database file is C<cpandb.sql>.
=back
There are two main methods available.
=head2 C<$obj-E<gt>index(%args);>
This is used to set up and maintain the database. The
following arguments are accepted:
=over 3
=item * setup =E<gt> 1
This specifies that the database is to be created and
populated from the CPAN indices; any existing database
will be overwritten. Not specifying this option will
assume that an existing database is to be updated.
=item * reindex =E<gt> 'dist_name'
This specifies that the CPAN distribution C<dist_name>
is to be reindexed.
=back
=head2 C<$obj-E<gt>query(%args);>
This is used for querying the database by distribution
name, module name, or CPAN author name. There are
two arguments needed to specify such queries.
=over 3
=item * C<mode =E<gt> some_value>
This specifies what type of query to perform,
with C<mode> being one of C<dist>, C<module>,
or C<author>, for searching through, respectively,
CPAN distribution names, module names, or author names and
CPAN ids.
=item * C<type =E<gt> query_term>
This specifies the query term for the search, with
C<type> being one of C<name>, to search for an
exact match, or C<search>, for searching for partial
matches. Perl regular expressions are supported in
the C<query_term> for the C<search> option.
=back
As well, an option of C<max_results =E<gt> some_number> will
limit the number of results returned; if not specified,
the limit doesn't apply.
=head1 CPAN.pm support
As of CPAN.pm version 1.88_65, there is experimental support
within CPAN.pm for using CPAN::SQLite to obtain
information on packages, modules, and authors. One goal
of this is to reduce the memory footprint of the CPAN.pm
shell, as this information is no longer all preloaded into
memory. This can be enabled through
perl -MCPAN -e shell
cpan> o conf use_sqlite 1
Use
cpan> o conf commit
to save this setting for future sessions.
Using CPAN::SQLite, what happens is that a request for information
through CPAN.pm, such as
cpan> a ANDK
will cause a query to the SQLite database to be made.
If successful, this will place the relevant data for this
request into the data structure CPAN.pm uses to store and
retrieve such information. Thus, at any given time, the
only information CPAN.pm stores in memory is that for
packages, modules, and authors for which previous queries
have been made. There are certain requests, such as
cpan> r
to make a list of recommended packages for which upgrades
on CPAN are available, which will result in loading
information on all available packages into memory; if such
a query is made, the subsequent memory footprint of CPAN.pm
with and without CPAN::SQLite will be essentially the same.
The database itself, called F<cpandb.sql>, will be stored
in the location specified by C<$CPAN::Config-E<gt>{cpan_home}>.
When first started, this database will be created, and afterwards,
it will be updated if the database is older than one day since
the last update. A log file of the creation or update process, called
F<cpan_search_log.dddddddddd>, will be created in the same
directory as the database file.
=head1 SEE ALSO
L<CPAN::SQLite::Index>, for setting up and maintaining the database, and
L<CPAN::SQLite::Search> for an interface to querying the database. Some
details of the interaction with L<CPAN> is available from
L<CPAN::SQLite::META>. See also the L<cpandb> script for a command-line
interface to the indexing and querying of the database.
=head1 SUPPORT
You can find documentation for this module with the perldoc command.
perldoc CPAN::SQLite
You can also look for information at:
=over 4
=item * AnnoCPAN: Annotated CPAN documentation
L<http://annocpan.org/dist/CPAN-SQLite>
=item * CPAN::Forum: Discussion forum
L<http:///www.cpanforum.com/dist/CPAN-SQLite>
=item * CPAN Ratings
L<http://cpanratings.perl.org/d/CPAN-SQLite>
=item * RT: CPAN's request tracker
L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=CPAN-SQLite>
=item * Search CPAN
L<http://search.cpan.org/dist/CPAN-SQLite>
=back
=head1 BUGS
At this time, CPAN::SQLite keeps information contained only
in the latest version of a CPAN distribution. This means that
modules that are provided only in older versions of a CPAN
distribution will not be present in the database; for example,
at this time, the latest version of the I<libwww-perl> distribution
on CPAN is 5.805, but there are modules such as I<URI::URL::finger>
contained in version 5.10 of libwww-perl that are not present in 5.805.
This behaviour differs from that of L<CPAN> without CPAN::SQLite.
This may change in the future.
Please report bugs and feature requests via
L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=CPAN-SQLite>.
=head1 ENVIRONMENT VARIABLES
Information messages from the indexing procedures are printed
out to STDOUT if the environment variable CPAN_SQLITE_DEBUG
is set. This is automatically set within L<CPAN::SQLite::Index>.
If CPAN_SQLITE_NO_LOG_FILES is set, no log files will be created
during the indexing procedures. Log files are deleted automatically
in 30 days. To override this, set CPAN_SQLITE_LOG_FILES_CLEANUP.
To stop automatic cleanup, set this variable to 0.
If CPAN_SQLITE_DOWNLOAD variable are set, an already existing and
up-to-date cpandb.sql file will be downloaded from
http://cpansqlite.trouchelle.com/ where it's updated every hour. This
greatly increases performance and decreases CPU and memory consumption
during the indexing process.
See L<CPAN::SQLite::Index> for more details, potential problems, and more
configuration options.
=head1 AUTHORS
Randy Kobes (passed away on September 18, 2010)
Serguei Trouchelle E<lt>stro@cpan.orgE<gt>
=head1 COPYRIGHT
Copyright 2006,2008 by Randy Kobes E<lt>r.kobes@uwinnipeg.caE<gt>.
Copyright 2011-2014 by Serguei Trouchelle E<lt>stro@cpan.orgE<gt>.
Use and redistribution are under the same terms as Perl itself.
=cut

View File

@@ -0,0 +1,169 @@
# $Id: DBI.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::DBI;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
require File::Spec;
use DBI;
use parent 'Exporter';
our ($dbh, $tables, @EXPORT_OK);
@EXPORT_OK = qw($dbh $tables);
$tables = {
'info' => {
'primary' => {
'status' => q!INTEGER NOT NULL PRIMARY KEY!,
},
'other' => {},
'key' => [],
'name' => 'status',
'id' => 'status',
},
mods => {
primary => { mod_id => q{INTEGER NOT NULL PRIMARY KEY} },
other => {
mod_name => q{VARCHAR(100) NOT NULL},
dist_id => q{INTEGER NOT NULL},
mod_abs => q{TEXT},
mod_vers => q{VARCHAR(10)},
},
key => [qw/dist_id mod_name/],
name => 'mod_name',
id => 'mod_id',
has_a => { dists => 'dist_id' },
},
dists => {
primary => { dist_id => q{INTEGER NOT NULL PRIMARY KEY} },
other => {
dist_name => q{VARCHAR(90) NOT NULL},
auth_id => q{INTEGER NOT NULL},
dist_file => q{VARCHAR(110) NOT NULL},
dist_vers => q{VARCHAR(20)},
dist_abs => q{TEXT},
},
key => [qw/auth_id dist_name/],
name => 'dist_name',
id => 'dist_id',
has_a => { auths => 'auth_id' },
has_many => { mods => 'dist_id', },
},
auths => {
primary => { auth_id => q{INTEGER NOT NULL PRIMARY KEY} },
other => {
cpanid => q{VARCHAR(20) NOT NULL},
fullname => q{VARCHAR(40) NOT NULL},
email => q{TEXT},
},
key => [qw/cpanid/],
has_many => { dists => 'dist_id' },
name => 'cpanid',
id => 'auth_id',
},
};
sub new {
my ($class, %args) = @_;
my $db_dir = $args{db_dir} || $args{CPAN};
my $db = File::Spec->catfile($db_dir, $args{db_name});
$dbh ||= DBI->connect(
"DBI:SQLite:$db",
'', '',
{
RaiseError => 1,
AutoCommit => 0,
sqlite_use_immediate_transaction => 0,
});
die "Cannot connect to $db" unless $dbh;
$dbh->{AutoCommit} = 0;
my $objs;
foreach my $table (keys %$tables) {
my $cl = $class . '::' . $table;
$objs->{$table} = $cl->make(table => $table);
}
for my $table (keys %$tables) {
foreach my $type (qw(primary other)) {
foreach my $column (keys %{ $tables->{$table}->{$type} }) {
push @{ $tables->{$table}->{columns} }, $column;
}
}
}
return bless { objs => $objs }, $class;
}
sub make {
my ($class, %args) = @_;
my $table = $args{table};
die qq{No table exists corresponding to '$class'} unless $table;
my $info = $tables->{$table};
die qq{No information available for table '$table'} unless $info;
my $self = {
table => $table,
columns => $info->{columns},
id => $info->{id},
name => $info->{name},
};
foreach (qw(name has_a has_many)) {
next unless defined $info->{$_};
$self->{$_} = $info->{$_};
}
return bless $self, $class;
}
sub db_error {
my ($obj, $sth) = @_;
return unless $dbh;
if ($sth) {
$sth->finish;
undef $sth;
}
return $obj->{error_msg} = q{Database error: } . $dbh->errstr;
}
1;
=head1 NAME
CPAN::SQLite::DBI - DBI information for the CPAN::SQLite database
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module is used by L<CPAN::SQLite::Index> and
L<CPAN::SQLite::Search> to set up some basic database
information. It exports two variables:
=over 3
=item C<$tables>
This is a hash reference whose keys are the table names, with
corresponding values being hash references whose keys are the
columns of the table and values being the associated data types.
=item C<$dbh>
This is a L<DBI> database handle used to connect to the
database.
=back
The main method of this module is C<make>, which is used
to make the tables of the database.
=head1 SEE ALSO
L<CPAN::SQLite::Index> and L<CPAN::SQLite::Search>
=cut

View File

@@ -0,0 +1,238 @@
# $Id: Index.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::DBI::Index;
use strict;
use warnings;
BEGIN {
our $VERSION = '0.219';
$CPAN::SQLite::DBI::Index::info::VERSION = $VERSION;
$CPAN::SQLite::DBI::Index::mods::VERSION = $VERSION;
$CPAN::SQLite::DBI::Index::dists::VERSION = $VERSION;
$CPAN::SQLite::DBI::Index::auths::VERSION = $VERSION;
}
use CPAN::SQLite::DBI qw($dbh);
use parent 'CPAN::SQLite::DBI';
package CPAN::SQLite::DBI::Index::info;
use parent 'CPAN::SQLite::DBI::Index';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Index::mods;
use parent 'CPAN::SQLite::DBI::Index';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Index::dists;
use parent 'CPAN::SQLite::DBI::Index';
use CPAN::SQLite::DBI qw($dbh);
sub fetch_ids {
my $self = shift;
my $sql = sprintf(qq{SELECT %s,%s,%s FROM %s}, $self->{id}, $self->{name}, 'dist_vers', $self->{table});
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
$sth->execute() or do {
$self->db_error($sth);
return;
};
my ($ids, $versions);
while (my ($id, $key, $vers) = $sth->fetchrow_array()) {
$ids->{$key} = $id;
$versions->{$key} = $vers;
}
$sth->finish;
undef $sth;
return ($ids, $versions);
}
package CPAN::SQLite::DBI::Index::auths;
use parent 'CPAN::SQLite::DBI::Index';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Index;
use CPAN::SQLite::DBI qw($tables);
use CPAN::SQLite::DBI qw($dbh);
sub fetch_ids {
my $self = shift;
my $sql = sprintf(qq{SELECT %s,%s from %s}, $self->{id}, $self->{name}, $self->{table});
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
$sth->execute() or do {
$self->db_error($sth);
return;
};
my $ids;
while (my ($id, $key) = $sth->fetchrow_array()) {
$ids->{$key} = $id;
}
$sth->finish;
undef $sth;
return $ids;
}
sub schema {
my ($self, $data) = @_;
my $schema = '';
foreach my $type (qw(primary other)) {
foreach my $column (keys %{ $data->{$type} }) {
$schema .= $column . ' ' . $data->{$type}->{$column} . ", ";
}
}
$schema =~ s{, $}{};
return $schema;
}
sub create_index {
my ($self, $data) = @_;
my $key = $data->{key};
my $table = $self->{table};
return 1 unless (defined $key and ref($key) eq 'ARRAY');
foreach my $index (@$key) {
my $id_name = 'ix_' . $table . '_' . $index;
$id_name =~ s/\(\s*\d+\s*\)//;
my $sql = 'CREATE INDEX ' . $id_name . ' ON ' . $table . '( ' . $index . ' )';
my $sth = $dbh->prepare($sql);
$sth->execute() or do {
$self->db_error($sth);
return;
};
$sth->finish;
undef $sth;
}
return 1;
}
sub drop_table {
my $self = shift;
my $table = $self->{table};
my $sql = qq{SELECT name FROM sqlite_master } . qq{ WHERE type='table' AND name=?};
my $sth = $dbh->prepare($sql);
$sth->execute($table);
if (defined $sth->fetchrow_array) {
$dbh->do(qq{drop table $table}) or do {
$self->db_error($sth);
return;
};
}
$sth->finish;
undef $sth;
return 1;
}
sub create_table {
my ($self, $schema) = @_;
return unless $schema;
my $sql = sprintf(qq{CREATE TABLE %s (%s)}, $self->{table}, $schema);
my $sth = $dbh->prepare($sql);
$sth->execute() or do {
$self->db_error($sth);
return;
};
$sth->finish;
undef $sth;
return 1;
}
sub create_tables {
my ($self, %args) = @_;
return unless $args{setup};
my $objs = $self->{objs};
foreach my $table (keys %$objs) {
next unless my $schema = $self->schema($tables->{$table});
my $obj = $objs->{$table};
$obj->drop_table or return;
$obj->create_table($schema) or return;
$obj->create_index($tables->{$table}) or return;
}
return 1;
}
sub sth_insert {
my ($self, $fields) = @_;
my $flds = join ',', @{$fields};
my $vals = join ',', map { '?' } @{$fields};
my $sql = sprintf(qq{INSERT INTO %s (%s) VALUES (%s)}, $self->{table}, $flds, $vals);
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
return $sth;
}
sub sth_update {
my ($self, $fields, $id, $rep_id) = @_;
my $set = join ',', map { "$_=?" } @{$fields};
my $sql = sprintf(qq{UPDATE %s SET %s WHERE %s = %s}, $self->{table}, $set, $self->{id}, $id);
$sql .= qq { AND rep_id = $rep_id } if ($rep_id);
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
return $sth;
}
sub sth_delete {
my ($self, $table_id, $rep_id) = @_;
my $sql = sprintf(qq{DELETE FROM %s where %s = ?}, $self->{table}, $table_id);
$sql .= qq { AND rep_id = $rep_id } if ($rep_id);
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
return $sth;
}
1;
=head1 NAME
CPAN::SQLite::DBI::Index - DBI information for indexing the CPAN::SQLite database
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module provides various methods for L<CPAN::SQLite::Index> in
indexing and populating the database from the index files.
=over
=item C<create_tables>
This creates the database tables.
=item C<drop_table>
This drops a table.
=item C<sth_insert>
This returns an C<$sth> statement handle for inserting
values into a table.
=item C<sth_update>
This returns an C<$sth> statement handle for updating
values into a table.
=item C<sth_delete>
This returns an C<$sth> statement handle for deleting
values from a table.
=back
=head1 SEE ALSO
L<CPAN::SQLite::Index>
=cut

View File

@@ -0,0 +1,283 @@
# $Id: Search.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::DBI::Search;
use strict;
use warnings;
BEGIN {
our $VERSION = '0.219';
$CPAN::SQLite::DBI::Search::info::VERSION = $VERSION;
$CPAN::SQLite::DBI::Search::mods::VERSION = $VERSION;
$CPAN::SQLite::DBI::Search::dists::VERSION = $VERSION;
$CPAN::SQLite::DBI::Search::auths::VERSION = $VERSION;
}
use parent 'CPAN::SQLite::DBI';
use CPAN::SQLite::DBI qw($tables $dbh);
use CPAN::SQLite::Util qw($full_id);
package CPAN::SQLite::DBI::Search::info;
use parent 'CPAN::SQLite::DBI::Search';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Search::mods;
use parent 'CPAN::SQLite::DBI::Search';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Search::dists;
use parent 'CPAN::SQLite::DBI::Search';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Search::auths;
use parent 'CPAN::SQLite::DBI::Search';
use CPAN::SQLite::DBI qw($dbh);
package CPAN::SQLite::DBI::Search;
use parent 'CPAN::SQLite::DBI';
use CPAN::SQLite::DBI qw($tables $dbh);
use CPAN::SQLite::Util qw($full_id download);
sub fetch {
my ($self, %args) = @_;
my $fields = $args{fields};
my $search = $args{search};
my @fields = ref($fields) eq 'ARRAY' ? @{$fields} : ($fields);
my $sql = $self->sql_statement(%args) or do {
$self->{error} = 'Error constructing sql statement: ' . $self->{error};
return;
};
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
$sth->execute() or do {
$self->db_error($sth);
return;
};
if (not $search->{wantarray}) {
my (%results, $results);
@results{@fields} = $sth->fetchrow_array;
$results = ($sth->rows == 0) ? undef : \%results;
$sth->finish;
undef $sth;
$self->extra_info($results) if $results;
return $results;
} else {
my (%hash, $results);
while (@hash{@fields} = $sth->fetchrow_array) {
my %tmp = %hash;
$self->extra_info(\%tmp);
push @{$results}, \%tmp;
}
$results = undef if ($sth->rows == 0);
$sth->finish;
undef $sth;
return $results;
}
}
sub fetch_and_set {
my ($self, %args) = @_;
my $fields = $args{fields};
my $search = $args{search};
my $meta_obj = $args{meta_obj};
die "Please supply a CPAN::SQLite::Meta::* object"
unless ($meta_obj and ref($meta_obj) =~ /^CPAN::SQLite::META/);
my @fields = ref($fields) eq 'ARRAY' ? @{$fields} : ($fields);
my $sql = $self->sql_statement(%args) or do {
$self->{error} = 'Error constructing sql statement: ' . $self->{error};
return;
};
my $sth = $dbh->prepare($sql) or do {
$self->db_error();
return;
};
$sth->execute() or do {
$self->db_error($sth);
return;
};
my $want_ids = $args{want_ids};
my $set_list = $args{set_list};
my $download = $args{download};
if (not $search->{wantarray}) {
my (%results, %meta_results, $results);
@results{@fields} = $sth->fetchrow_array;
$results = ($sth->rows == 0) ? undef : \%results;
$sth->finish;
undef $sth;
return unless $results;
$self->extra_info($results);
$meta_obj->set_data($results);
if ($want_ids) {
$meta_results{dist_id} = $results{dist_id};
$meta_results{download} = download($results{cpanid}, $results{dist_file});
return \%meta_results;
} else {
return 1;
}
} else {
my (%hash, $meta_results);
while (@hash{@fields} = $sth->fetchrow_array) {
my %tmp = %hash;
if ($set_list) {
push @{$meta_results}, \%tmp;
} else {
$self->extra_info(\%tmp);
$meta_obj->set_data(\%tmp);
if ($want_ids) {
my $download = download($tmp{cpanid}, $tmp{dist_file});
push @{$meta_results},
{
dist_id => $tmp{dist_id},
download => $download
};
}
}
}
$meta_results = undef if ($sth->rows == 0);
$sth->finish;
undef $sth;
return unless $meta_results;
$meta_obj->set_list_data($meta_results, $download) if $set_list;
return $want_ids ? $meta_results : 1;
}
}
sub extra_info {
my ($self, $results) = @_;
if ($results->{cpanid} and $results->{dist_file}) {
$results->{download} = download($results->{cpanid}, $results->{dist_file});
}
return;
}
sub sql_statement {
my ($self, %args) = @_;
my $search = $args{search};
my $distinct = $search->{distinct} ? 'DISTINCT' : '';
my $table = $args{table};
my $fields = $args{fields};
my @fields = ref($fields) eq 'ARRAY' ? @{$fields} : ($fields);
for (@fields) {
$_ = $full_id->{$_} if $full_id->{$_};
}
my $sql = qq{SELECT $distinct } . join(',', @fields);
my $where = '';
my $type = $search->{type};
QUERY: {
($type eq 'query') and do {
my $value = $search->{value};
last QUERY if ($value eq '^');
my $name = $search->{name};
my $text = $search->{text};
my $use_like = ($value =~ /^\^?[A-Za-z0-9_\\\:\-]+$/) ? 1 : 0;
my $prepend = '%';
if ($use_like and $value =~ /^\^/) {
$prepend = '';
$value =~ s/^\^//;
$value =~ s{\\}{}g;
}
$where =
$use_like
? qq{$name LIKE '$prepend$value%'}
: qq{$name REGEXP '(?i:$value)'};
if ($name eq 'cpanid') {
$where .=
$use_like
? qq{ OR $text LIKE '$prepend$value%'}
: qq{ OR $text REGEXP '(?i:$value)'};
}
last QUERY;
};
($type eq 'id') and do {
$where = qq{ $search->{id} = $search->{value} };
last QUERY;
};
($type eq 'name') and do {
$where = qq{ $search->{name} = '$search->{value}' };
last QUERY;
};
warn qq{Unknown query type};
return;
}
my $join;
$sql .= ' FROM ' . $table;
my $left_join = $args{join} || $args{left_join};
if ($left_join) {
if (ref($left_join) eq 'HASH') {
foreach my $key (keys %$left_join) {
my $id = $left_join->{$key};
$sql .= " LEFT JOIN $key ON $table.$id=$key.$id ";
}
}
}
if ($where) {
$sql .= ' WHERE ( ' . $where . ' )';
$sql .= ' AND (' . $join . ')' if $join;
} else {
$sql .= ' WHERE (' . $join . ')' if $join;
}
my $order_by = '';
if (my $user_order_by = $args{order_by}) {
$order_by = $order_by ? "$order_by,$user_order_by" : $user_order_by;
}
if ($order_by and $where) {
$sql .= qq{ ORDER BY $order_by };
}
if (my $limit = $args{limit}) {
my ($min, $max) =
ref($limit) eq 'HASH'
? ($limit->{min} || 0, $limit->{max})
: (0, $limit);
$sql .= qq{ LIMIT $min,$max };
}
return $sql;
}
1;
=head1 NAME
CPAN::SQLite::DBI::Search - DBI information for searching the CPAN::SQLite database
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module provides methods for L<CPAN::SQLite::Search> for searching
the C<CPAN::SQLite> database. There are two main methods.
=over
=item C<fetch>
This takes information from C<CPAN::SQLite::Search> and sets up
a query on the database, returning the results found.
=item C<sql_statement>
This is used by the C<fetch> method to construct the appropriate
SQL statement.
=back
=head1 SEE ALSO
L<CPAN::SQLite::Search>
=cut

View File

@@ -0,0 +1,379 @@
# $Id: Index.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::Index;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use CPAN::SQLite::Info;
use CPAN::SQLite::State;
use CPAN::SQLite::Populate;
use CPAN::SQLite::DBI qw($tables);
use File::Spec::Functions qw(catfile);
use File::Basename;
use File::Path;
use HTTP::Tiny;
use Scalar::Util 'weaken';
unless ($ENV{CPAN_SQLITE_NO_LOG_FILES}) {
$ENV{CPAN_SQLITE_DEBUG} = 1;
}
our ($oldout);
my $log_file = 'cpan_sqlite_log.' . time;
# This is usually already defined in real life, but tests need it to be set
$CPAN::FrontEnd ||= "CPAN::Shell";
sub new {
my ($class, %args) = @_;
if ($args{setup} and $args{reindex}) {
die "Reindexing must be done on an exisiting database";
}
my $self = { index => undef, state => undef, %args };
return bless $self, $class;
}
sub download_index {
my $self = shift;
if ($ENV{'CPAN_SQLITE_DOWNLOAD'}) {
$ENV{'CPAN_SQLITE_DOWNLOAD_URL'} = 'http://cpansqlite.trouchelle.com/' unless $ENV{'CPAN_SQLITE_DOWNLOAD_URL'};
}
return 0 unless $ENV{'CPAN_SQLITE_DOWNLOAD_URL'};
$CPAN::FrontEnd->myprint("Downloading the compiled index db ... ");
if (my $response =
HTTP::Tiny->new->mirror($ENV{'CPAN_SQLITE_DOWNLOAD_URL'} => catfile($self->{'db_dir'}, $self->{'db_name'})))
{
if ($response->{'success'} and $response->{'status'} and $response->{'status'} eq '200') {
if (my $type = $response->{'headers'}->{'content-type'}) {
if ($type eq 'application/x-sqlite3') {
return 1;
}
}
}
}
$CPAN::FrontEnd->mywarn('Cannot download the compiled index db');
return 0;
}
sub index {
my $self = shift;
my $setup = $self->{'setup'};
if ($setup) {
my $db_name = catfile($self->{'db_dir'}, $self->{db_name});
if (-f $db_name) {
$CPAN::FrontEnd->myprint("Removing existing $db_name ... ");
if (unlink $db_name) {
$CPAN::FrontEnd->myprint("Done.\n");
} else {
$CPAN::FrontEnd->mywarn("Failed: $!\n");
}
}
}
my $log = catfile($self->{'log_dir'}, $log_file);
unless ($ENV{'CPAN_SQLITE_NO_LOG_FILES'}) {
$oldout = error_fh($log);
}
my $log_cleanup = $ENV{'CPAN_SQLITE_LOG_FILES_CLEANUP'};
$log_cleanup = 30 unless defined $log_cleanup;
if ($log_cleanup and $log_cleanup =~ /^\d+$/) {
if (opendir(my $DIR, $self->{'log_dir'})) {
my @files = grep { /cpan_sqlite_log\./ } readdir $DIR;
closedir $DIR;
@files = grep { -C $_ > $log_cleanup } map { catfile($self->{'log_dir'}, $_) } @files;
if (@files) {
$CPAN::FrontEnd->myprint('Cleaning old log files ... ');
unlink @files;
$CPAN::FrontEnd->myprint("Done.\n");
}
}
}
if ($self->download_index()) {
return 1;
}
if ($self->{'update_indices'}) {
$CPAN::FrontEnd->myprint('Fetching index files ... ');
if ($self->fetch_cpan_indices()) {
$CPAN::FrontEnd->myprint("Done.\n");
} else {
$CPAN::FrontEnd->mywarn("Failed\n");
return;
}
}
$CPAN::FrontEnd->myprint('Gathering information from index files ... ');
if ($self->fetch_info()) {
$CPAN::FrontEnd->myprint("Done.\n");
} else {
$CPAN::FrontEnd->mywarn("Failed\n");
return;
}
unless ($setup) {
$CPAN::FrontEnd->myprint('Obtaining current state of database ... ');
if ($self->state()) {
$CPAN::FrontEnd->myprint("Done.\n");
} else {
$CPAN::FrontEnd->mywarn("Failed\n");
return;
}
}
$CPAN::FrontEnd->myprint('Populating database tables ... ');
if ($self->populate()) {
$CPAN::FrontEnd->myprint("Done.\n");
} else {
$CPAN::FrontEnd->mywarn("Failed\n");
return;
}
return 1;
}
sub fetch_cpan_indices {
my $self = shift;
my $CPAN = $self->{CPAN};
my $indices = {
'01mailrc.txt.gz' => 'authors',
'02packages.details.txt.gz' => 'modules',
};
foreach my $index (keys %$indices) {
my $file = catfile($CPAN, $indices->{$index}, $index);
next if (-e $file and -M $file < 1);
my $dir = dirname($file);
unless (-d $dir) {
mkpath($dir, 0, oct(755)) or die "Cannot mkpath $dir: $!";
}
my @urllist = @{ $self->{urllist} };
foreach my $cpan (@urllist) {
my $from = join '/', ($cpan, $indices->{$index}, $index);
if (my $response = HTTP::Tiny->new->get($from)) {
if ($response->{'success'}) {
if (open(my $FILE, '>', $file)) {
binmode $FILE;
print $FILE $response->{'content'};
if (close($FILE)) {
next;
}
}
}
}
}
unless (-f $file) {
$CPAN::FrontEnd->mywarn("Cannot retrieve '$file'");
return;
}
}
return 1;
}
sub fetch_info {
my $self = shift;
my %wanted = map { $_ => $self->{$_} } qw(CPAN ignore keep_source_where);
my $info = CPAN::SQLite::Info->new(%wanted);
$info->fetch_info() or return;
my @tables = qw(dists mods auths info);
my $index;
foreach my $table (@tables) {
my $class = __PACKAGE__ . '::' . $table;
my $this = { info => $info->{$table} };
$index->{$table} = bless $this, $class;
}
$self->{index} = $index;
return 1;
}
sub state {
my $self = shift;
my %wanted = map { $_ => $self->{$_} } qw(db_name index setup reindex db_dir);
my $state = CPAN::SQLite::State->new(%wanted);
$state->state() or return;
$self->{state} = $state;
return 1;
}
sub populate {
my $self = shift;
my %wanted = map { $_ => $self->{$_} } qw(db_name index setup state db_dir);
my $db = CPAN::SQLite::Populate->new(%wanted);
$db->populate() or return;
return 1;
}
sub error_fh {
my $file = shift;
open(my $tmp, '>', $file) or die "Cannot open $file: $!";
close $tmp;
# Should be open(my $oldout, '>&', \*STDOUT); but it fails on 5.6.2
open(my $oldout, '>&STDOUT');
open(STDOUT, '>', $file) or die "Cannot tie STDOUT to $file: $!";
select STDOUT;
$| = 1;
return $oldout;
}
sub DESTROY {
unless ($ENV{CPAN_SQLITE_NO_LOG_FILES}) {
close STDOUT;
open(STDOUT, '>&', $oldout) if $oldout;
}
return;
}
1;
=head1 NAME
CPAN::SQLite::Index - set up or update database tables.
=head1 VERSION
version 0.219
=head1 SYNOPSIS
my $index = CPAN::SQLite::Index->new(setup => 1);
$index->index();
=head1 DESCRIPTION
This is the main module used to set up or update the
database tables used to store information from the
CPAN and ppm indices. The creation of the object
my $index = CPAN::SQLite::Index->new(%args);
accepts two possible arguments:
=over 3
=item * setup =E<gt> 1
This (optional) argument specifies that the database is being set up.
Any existing tables will be dropped.
=item * reindex =E<gt> value
This (optional) argument specifies distribution names that
one would like to reindex in an existing database. These may
be specified as either a scalar, for a single distribution,
or as an array reference for a list of distributions.
=back
=head1 DETAILS
Calling
$index->index();
will start the indexing procedure. Various messages
detailing the progress will written to I<STDOUT>,
which by default will be captured into a file
F<cpan_sqlite_log.dddddddddd>, where the extension
is the C<time> that the method was invoked. Error messages
are not captured, and will appear in I<STDERR>.
The steps of the indexing procedure are as follows.
=over 4
=item * download existing pre-compiled index (optional)
If CPAN_SQLITE_DOWNLOAD or CPAN_SQLITE_DOWNLOAD_URL variables are set, an
already existing and up-to-date cpandb.sql file will be downloaded from
either specified URL or http://cpansqlite.trouchelle.com/ where it's
updated every hour. This greatly increases performance and decreases CPU
and memory consumption during the indexing process but if your CPAN
mirror is out-of-sync or you're using DarkPAN, it obviously wouldn't
work. It also wouldn't work without an internet connection.
See L<WWW::CPAN::SQLite> if you want to setup your own service for
pre-compiling the database.
If neither variable is set, this step is skipped.
=item * fetch index data
The necessary CPAN index files
F<$CPAN/authors/01mailrc.txt.gz> and
F<$CPAN/modules/02packages.details.txt.gz> will be fetched
from the CPAN mirror specified by the C<$cpan> variable
at the beginning of L<CPAN::SQLite::Index>. If you are
using this option, it is recommended to use the
same CPAN mirror with subsequent updates, to ensure consistency
of the database. As well, the information on the locations
of the CPAN mirrors used for Template-Toolkit and GeoIP
is written.
=item * get index information
Information from the CPAN indices is extracted through
L<CPAN::SQLite::Info>.
=item * get state information
Unless the C<setup> argument within the C<new>
method of L<CPAN::SQLite::Index> is specified,
this will get information on the state of the database
through L<CPAN::SQLite::State>.
A comparison is then made between this information
and that gathered from the CPAN indices, and if there's
a discrepancy in some items, those items are marked
for either insertion, updating, or deletion, as appropriate.
=item * populate the database
At this stage the gathered information is used to populate
the database, through L<CPAN::SQLite::Populate>,
either inserting new items, updating
existing ones, or deleting obsolete items.
=back
=head1 SEE ALSO
L<CPAN::SQLite::Info>, L<CPAN::SQLite::State>,
L<CPAN::SQLite::Populate>,
and L<CPAN::SQLite::Util>.
Development takes place on the CPAN-SQLite project
at L<http://sourceforge.net/projects/cpan-search/>.
=head1 AUTHORS
Randy Kobes (passed away on September 18, 2010)
Serguei Trouchelle E<lt>stro@cpan.orgE<gt>
=head1 COPYRIGHT
Copyright 2006 by Randy Kobes E<lt>r.kobes@uwinnipeg.caE<gt>.
Copyright 2011 by Serguei Trouchelle E<lt>stro@cpan.orgE<gt>.
Use and redistribution are under the same terms as Perl itself.
=cut

View File

@@ -0,0 +1,280 @@
# $Id: Info.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::Info;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use CPAN::DistnameInfo;
use File::Spec::Functions qw(catfile);
use Compress::Zlib;
use File::Basename;
use Safe;
use CPAN::SQLite::Util qw(vcmp print_debug);
my $ext = qr/\.(tar\.gz|tar\.Z|tgz|zip)$/;
sub new {
my ($class, %args) = @_;
my $self = { dists => {}, auths => {}, mods => {}, info => {}, %args };
return bless $self, $class;
}
sub fetch_info {
my $self = shift;
$self->mailrc() or return;
$self->dists_and_mods() or return;
return 1;
}
sub dists_and_mods {
my $self = shift;
my ($packages, $cpan_files) = $self->packages();
my ($dists, $mods);
my $ignore = $self->{ignore};
my $pat;
if ($ignore and ref($ignore) eq 'ARRAY') {
$pat = join '|', @$ignore;
}
foreach my $cpan_file (keys %$cpan_files) {
if ($pat and ($cpan_file =~ /^($pat)/)) {
delete $cpan_files->{$cpan_file};
print_debug("Ignoring $cpan_file\n");
next;
}
my $d = CPAN::DistnameInfo->new($cpan_file);
next unless ($d->maturity eq 'released');
my $dist_name = $d->dist;
my $dist_vers = $d->version;
my $cpanid = $d->cpanid;
my $dist_file = $d->filename;
unless ($dist_name and $dist_vers and $cpanid) {
print_debug("No dist_name/version/cpanid for $cpan_file: skipping\n");
delete $cpan_files->{$cpan_file};
next;
}
# ignore specified dists
if ($pat and ($dist_name =~ /^($pat)$/)) {
delete $cpan_files->{$cpan_file};
print_debug("Ignoring $dist_name\n");
next;
}
if (not $dists->{$dist_name} or vcmp($dist_vers, $dists->{$dist_name}->{dist_vers}) > 0) {
$dists->{$dist_name}->{dist_vers} = $dist_vers;
$dists->{$dist_name}->{dist_file} = $dist_file;
$dists->{$dist_name}->{cpanid} = $cpanid;
}
}
my $wanted;
foreach my $dist_name (keys %$dists) {
$wanted->{ basename($dists->{$dist_name}->{dist_file}) } = $dist_name;
}
foreach my $mod_name (keys %$packages) {
my $file = basename($packages->{$mod_name}->{dist_file});
my $dist_name = $wanted->{$file};
unless ($dist_name and $dists->{$dist_name}) {
delete $packages->{$mod_name};
next;
}
$mods->{$mod_name}->{dist_name} = $dist_name;
$dists->{$dist_name}->{modules}->{$mod_name}++;
$mods->{$mod_name}->{mod_vers} = $packages->{$mod_name}->{mod_vers};
}
$self->{dists} = $dists;
return $self->{mods} = $mods;
}
sub modlist {
my $self = shift;
warn 'Modlist does not contain any useful info anymore';
return;
}
sub packages {
my $self = shift;
my $index = 'modules/02packages.details.txt.gz';
my $packages =
$self->{keep_source_where}
? CPAN::FTP->localize($index, catfile($self->{keep_source_where}, $index))
: catfile($self->{CPAN}, $index);
return unless check_file('modules/02packages.details.txt.gz', $packages);
print_debug("Reading information from $packages\n");
my $lines = zcat($packages);
while (@$lines) {
my $shift = shift(@$lines);
last if $shift =~ /^\s*$/;
}
my ($mods, $cpan_files);
foreach (@$lines) {
my ($mod_name, $mod_vers, $dist_file) = split(" ", $_, 4);
$mod_vers = undef if $mod_vers eq 'undef';
$mods->{$mod_name} = { mod_vers => $mod_vers, dist_file => $dist_file };
$cpan_files->{$dist_file}++;
}
return ($mods, $cpan_files);
}
sub mailrc {
my $self = shift;
my $index = 'authors/01mailrc.txt.gz';
my $mailrc =
$self->{keep_source_where}
? CPAN::FTP->localize($index, catfile($self->{keep_source_where}, $index))
: catfile($self->{CPAN}, $index);
return unless check_file('authors/01mailrc.txt.gz', $mailrc);
print_debug("Reading information from $mailrc\n");
my $lines = zcat($mailrc);
my $auths;
foreach (@$lines) {
#my($cpanid,$fullname,$email) =
#m/alias\s+(\S+)\s+\"([^\"\<]+)\s+\<([^\>]+)\>\"/;
my ($cpanid, $authinfo) = m/alias\s+(\S+)\s+\"([^\"]+)\"/;
next unless $cpanid;
my ($fullname, $email);
if ($authinfo =~ m/([^<]+)\<(.*)\>/) {
$fullname = $1;
$email = $2;
} else {
$fullname = '';
$email = lc($cpanid) . '@cpan.org';
}
$auths->{$cpanid} = {
fullname => trim($fullname),
email => trim($email) };
}
return $self->{auths} = $auths;
}
sub check_file {
my ($index, $file) = @_;
unless ($file) {
warn qq{index file '$index' not defined};
return;
}
unless (-f $file) {
warn qq{index file '$file' not found};
return;
}
return 1;
}
sub zcat {
my $file = shift;
my ($buffer, $lines);
my $gz = gzopen($file, 'rb')
or die "Cannot open $file: $gzerrno";
while ($gz->gzreadline($buffer) > 0) {
push @$lines, $buffer;
}
die "Error reading from $file: $gzerrno" . ($gzerrno + 0)
if $gzerrno != Z_STREAM_END;
$gz->gzclose();
return $lines;
}
sub trim {
my $string = shift;
return '' unless $string;
$string =~ s/^\s+//;
$string =~ s/\s+$//;
$string =~ s/\s+/ /g;
return $string;
}
1;
=head1 NAME
CPAN::SQLite::Info - extract information from CPAN indices
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module extracts information from the CPAN indices
F<$CPAN/modules/02packages.details.txt.gz> and
F<$CPAN/authors/01mailrc.txt.gz>.
A C<CPAN::SQLite::Info> object is created with
my $info = CPAN::SQLite::Info->new(CPAN => $cpan);
where C<$cpan> specifies the top-level CPAN directory
underneath which the index files are found. Calling
$info->fetch_info();
will result in the object being populated with 3 hash references:
=over 3
=item * C<$info-E<gt>{dists}>
This contains information on distributions. Keys of this hash
reference are the distribution names, with the associated value being a
hash reference with keys of
=over 3
=item C<dist_vers> - the version of the CPAN file
=item C<dist_file> - the CPAN filename
=item C<cpanid> - the CPAN author id
=item C<dist_abs> - a description, if available
=item C<modules> - specifies the modules present in the distribution:
for my $module (keys %{$info->{$distname}->{modules}}) {
print "Module: $module\n";
}
=back
=item * C<$info-E<gt>{mods}>
This contains information on modules. Keys of this hash
reference are the module names, with the associated values being a
hash reference with keys of
=over 3
=item C<dist_name> - the distribution name containing the module
=item C<mod_vers> - the version
=item C<mod_abs> - a description, if available
=back
=item * C<$info-E<gt>{auths}>
This contains information on CPAN authors. Keys of this hash
reference are the CPAN ids, with the associated value being a
hash reference with keys of
=over 3
=item C<fullname> - the author's full name
=item C<email> - the author's email address
=back
=back
=head1 SEE ALSO
L<CPAN::SQLite::Index>
=cut

View File

@@ -0,0 +1,503 @@
# $Id: META.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::META;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
require CPAN::SQLite;
use DBI;
use File::Spec;
use parent 'Exporter';
our @EXPORT_OK;
@EXPORT_OK = qw(setup update check);
our $global_id;
# This is usually already defined in real life, but tests need it to be set
$CPAN::FrontEnd ||= "CPAN::Shell";
sub new {
my ($class, $cpan_meta) = @_;
my $cpan_sqlite = CPAN::SQLite->new();
return bless { cpan_meta => $cpan_meta, cpan_sqlite => $cpan_sqlite }, $class;
}
sub set {
my ($self, $class, $id) = @_;
my $sqlite_obj = $self->make_obj(class => $class, id => $id);
return $sqlite_obj->set_one();
}
sub search {
my ($self, $class, $regex) = @_;
my $sqlite_obj = $self->make_obj(class => $class, regex => $regex);
return $sqlite_obj->set_many();
}
sub make_obj {
my ($self, %args) = @_;
my $class = $args{class};
die qq{Must supply a CPAN::* class string}
unless ($class and $class =~ /^CPAN::/);
(my $type = $class) =~ s/^CPAN//;
my $package = __PACKAGE__ . $type;
return bless {
cpan_meta => $self->{cpan_meta},
cpan_sqlite => $self->{cpan_sqlite},
class => $class,
id => $args{id},
regex => $args{regex},
}, $package;
}
package CPAN::SQLite::META::Author;
use parent 'CPAN::SQLite::META';
use CPAN::SQLite::Util qw(has_hash_data);
sub set_one {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $id = $self->{id};
my $class = $self->{class};
$cpan_sqlite->{results} = {};
$cpan_sqlite->query(mode => 'author', name => $id, meta_obj => $self);
my $cpan_meta = $self->{cpan_meta};
return $cpan_meta->{readonly}{$class}{$id};
}
sub set_many {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $regex = $self->{regex};
$cpan_sqlite->{results} = [];
return $cpan_sqlite->query(mode => 'author', query => $regex, meta_obj => $self);
}
sub set_data {
my ($self, $results) = @_;
return $self->set_author($results->{cpanid}, $results);
}
package CPAN::SQLite::META::Distribution;
use parent 'CPAN::SQLite::META';
use CPAN::SQLite::Util qw(has_hash_data download);
use CPAN::DistnameInfo;
my $ext = qr{\.(tar\.gz|tar\.Z|tgz|zip)$};
sub set_one {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $id = $self->{id};
my ($dist_name, $dist_id);
if ($id =~ /$ext/) {
($dist_name, $dist_id) = $self->extract_distinfo($id);
}
return unless ($dist_name and $dist_id);
my $class = $self->{class};
$cpan_sqlite->{results} = {};
$cpan_sqlite->query(mode => 'dist', name => $dist_name, meta_obj => $self);
my $cpan_meta = $self->{cpan_meta};
return $cpan_meta->{readonly}{$class}{$dist_id};
}
sub set_many {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $regex = $self->{regex};
$cpan_sqlite->{results} = [];
return $cpan_sqlite->query(mode => 'dist', query => $regex, meta_obj => $self);
}
sub set_data {
my ($self, $results) = @_;
$global_id = $results->{download};
return $self->set_dist($results->{download}, $results);
}
sub set_list_data {
my ($self, $results, $download) = @_;
$global_id = $download;
$self->set_containsmods($results);
$global_id = undef;
return;
}
package CPAN::SQLite::META::Module;
use parent 'CPAN::SQLite::META';
use CPAN::SQLite::Util qw(has_hash_data);
sub set_one {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $id = $self->{id};
return if ($id =~ /^Bundle::/);
my $class = $self->{class};
$cpan_sqlite->{results} = {};
$cpan_sqlite->query(mode => 'module', name => $id, meta_obj => $self);
my $cpan_meta = $self->{cpan_meta};
return $cpan_meta->{readonly}{$class}{$id};
}
sub set_many {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $regex = $self->{regex};
$cpan_sqlite->{results} = [];
return $cpan_sqlite->query(mode => 'module', query => $regex, meta_obj => $self);
}
sub set_data {
my ($self, $results) = @_;
$self->set_module($results->{mod_name}, $results);
$global_id = $results->{download};
return $self->set_dist($results->{download}, $results);
}
sub set_list_data {
my ($self, $results, $download) = @_;
$global_id = $download;
$self->set_containsmods($results);
$global_id = undef;
return;
}
package CPAN::SQLite::META::Bundle;
use parent 'CPAN::SQLite::META';
use CPAN::SQLite::Util qw(has_hash_data);
sub set_one {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $id = $self->{id};
unless ($id =~ /^Bundle::/) {
$id = 'Bundle::' . $id;
}
my $class = $self->{class};
$cpan_sqlite->{results} = {};
$cpan_sqlite->query(mode => 'module', name => $id, meta_obj => $self);
my $cpan_meta = $self->{cpan_meta};
return $cpan_meta->{readonly}{$class}{$id};
}
sub set_many {
my $self = shift;
my $cpan_sqlite = $self->{cpan_sqlite};
my $regex = $self->{regex};
unless ($regex =~ /(^Bundle::|[\^\$\*\+\?\|])/i) {
$regex = '^Bundle::' . $regex;
}
$regex = '^Bundle::' if $regex eq '^';
$cpan_sqlite->{results} = [];
return $cpan_sqlite->query(mode => 'module', query => $regex, meta_obj => $self);
}
sub set_data {
my ($self, $results) = @_;
$self->set_bundle($results->{mod_name}, $results);
$global_id = $results->{download};
return $self->set_dist($results->{download}, $results);
}
sub set_list_data {
my ($self, $results, $download) = @_;
$global_id = $download;
$self->set_containsmods($results);
$global_id = undef;
return;
}
package CPAN::SQLite::META;
use CPAN::SQLite::Util qw(download);
my @months = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
my @days = qw(Sun Mon Tue Wed Thu Fri Sat);
sub set_author {
my ($self, $id, $results) = @_;
my $class = 'CPAN::Author';
my $cpan_meta = $self->{cpan_meta};
return $cpan_meta->instance($class => $id)->set(
'FULLNAME' => $results->{fullname},
'EMAIL' => $results->{email},
);
}
sub set_module {
my ($self, $id, $results) = @_;
my $class = 'CPAN::Module';
my $cpan_meta = $self->{cpan_meta};
my $d = $cpan_meta->instance($class => $id);
return $d->set(
'description' => $results->{mod_abs},
'userid' => $results->{cpanid},
'CPAN_VERSION' => $results->{mod_vers},
'CPAN_FILE' => $results->{download},
'CPAN_USERID' => $results->{cpanid},
);
}
sub set_bundle {
my ($self, $id, $results) = @_;
my $class = 'CPAN::Bundle';
my $cpan_meta = $self->{cpan_meta};
my $d = $cpan_meta->instance($class => $id);
return $d->set(
'description' => $results->{mod_abs},
'userid' => $results->{cpanid},
'CPAN_VERSION' => $results->{mod_vers},
'CPAN_FILE' => $results->{download},
'CPAN_USERID' => $results->{cpanid},
);
}
sub set_dist {
my ($self, $id, $results) = @_;
my $class = 'CPAN::Distribution';
my $cpan_meta = $self->{cpan_meta};
my $d = $cpan_meta->instance($class => $id);
return $d->set(
'DESCRIPTION' => $results->{dist_abs},
'CPAN_USERID' => $results->{cpanid},
'CPAN_VERSION' => $results->{dist_vers},
);
}
sub set_containsmods {
my ($self, $mods) = @_;
my $class = 'CPAN::Distribution';
my $cpan_meta = $self->{cpan_meta};
my %containsmods;
if ($mods and (ref($mods) eq 'ARRAY')) {
%containsmods = map { $_->{mod_name} => 1 } @$mods;
}
my $d = $cpan_meta->instance($class => $global_id);
return $d->{CONTAINSMODS} = \%containsmods;
}
sub reload {
my ($self, %args) = @_;
my $time = $args{'time'} || time;
my $force = $args{force};
my $db_name = $CPAN::SQLite::db_name;
my $db = File::Spec->catfile($CPAN::Config->{cpan_home}, $db_name);
my $journal_file = $db . '-journal';
if (-e $journal_file) {
$CPAN::FrontEnd->mywarn('Database locked - cannot update.');
return;
}
my @args = ($^X, '-MCPAN::SQLite::META=setup,update,check', '-e');
if (-e $db && -s _) {
my $mtime_db = (stat(_))[9];
my $time_string = gmtime_string($mtime_db);
$CPAN::FrontEnd->myprint("Database was generated on $time_string\n");
# Check for status, force update if it fails
if (system(@args, 'check')) {
$force = 1;
$CPAN::FrontEnd->myprint("Database file requires reindexing\n");
}
unless ($force) {
return if (($time - $mtime_db) < $CPAN::Config->{index_expire} * 86400);
}
$CPAN::FrontEnd->myprint('Updating database file ... ');
push @args, q{update};
} else {
unlink($db) if -e _;
$CPAN::FrontEnd->myprint('Creating database file ... ');
push @args, q{setup};
}
if ($CPAN::SQLite::DBI::dbh) {
$CPAN::SQLite::DBI::dbh->disconnect();
$CPAN::SQLite::DBI::dbh = undef;
}
system(@args) == 0 or die qq{system @args failed: $?};
$CPAN::FrontEnd->myprint("Done!\n");
return 1;
}
sub setup {
my $obj = CPAN::SQLite->new(setup => 1);
$obj->index() or die qq{CPAN::SQLite setup failed};
return;
}
sub update {
my $obj = CPAN::SQLite->new();
$obj->index() or die qq{CPAN::SQLite update failed};
return;
}
sub check {
my $obj = CPAN::SQLite->new();
my $db = File::Spec->catfile($obj->{'db_dir'}, $obj->{'db_name'});
my $dbh = DBI->connect("DBI:SQLite:$db", '', '', { 'RaiseError' => 0, 'PrintError' => 0, 'AutoCommit' => 1 });
if (my $sth = $dbh->prepare('SELECT status FROM info WHERE status = 1')) {
if ($sth->execute()) {
if ($sth->fetchrow_arrayref()) {
exit 0; # status = 1
} else {
exit 1; # status <> 1, need reindexing
}
} else {
# Something's wrong, will be safer to reinitialize
$dbh->disconnect();
undef $dbh;
setup();
update();
}
} else {
# Probably old version of DB or no DB at all, run setup and update
$dbh->disconnect();
undef $dbh;
setup();
update();
}
return;
}
sub gmtime_string {
my $time = shift;
return unless $time;
my @a = gmtime($time);
my $string =
sprintf("%s, %02d %s %d %02d:%02d:%02d GMT", $days[$a[6]], $a[3], $months[$a[4]], $a[5] + 1900, $a[2], $a[1], $a[0]);
return $string;
}
sub extract_distinfo {
my ($self, $pathname) = @_;
unless ($pathname =~ m{^\w/\w\w/}) {
$pathname =~ s{^(\w)(\w)(.*)}{$1/$1$2/$1$2$3};
}
my $d = CPAN::DistnameInfo->new($pathname);
my $dist = $d->dist;
my $download = download($d->cpanid, $d->filename);
return ($dist and $download) ? ($dist, $download) : undef;
}
1;
=head1 NAME
CPAN::SQLite::META - helper module for CPAN.pm integration
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module has no direct public interface, but is intended
as a helper module for use of CPAN::SQLite within the CPAN.pm
module. A new object is created as
my $obj = CPAN::SQLite::META->new($CPAN::META);
where C<$CPAN::META> comes from CPAN.pm. There are then
two main methods available.
=over 4
=item C<set>
This is used as
$obj->set($class, $id);
where C<$class> is one of C<CPAN::Author>, C<CPAN::Module>, or
C<CPAN::Distribution>, and C<$id> is the id CPAN.pm uses to
identify the class. The method searches the C<CPAN::SQLite>
database by name using the appropriate C<author>, C<dist>,
or C<module> mode, and if a result is found, calls
$CPAN::META->instance(
$class => $id
)->set(
%attributes
);
to register an instance of this class within C<CPAN.pm>.
=item C<ssearch>
This is used as
$obj->search($class, $id);
where C<$class> is one of C<CPAN::Author>, C<CPAN::Module>, or
C<CPAN::Distribution>, and C<$id> is the id CPAN.pm uses to
identify the class. The method searches the C<CPAN::SQLite>
database by C<query> using the appropriate C<author>, C<dist>,
or C<module> mode, and if results are found, calls
$CPAN::META->instance(
$class => $id
)->set(
%attributes
);
for each match to register an instance of this class
within C<CPAN.pm>.
=back
The attributes set within C<$CPAN::META->instance> depend
on the particular class.
=over
=item author
The attributes are
'FULLNAME' => $results->{fullname},
'EMAIL' => $results->{email},
where C<$results> are the results returned from C<CPAN::SQLite>.
=item module
The attributes are
'description' => $results->{mod_abs},
'userid' => $results->{cpanid},
'CPAN_VERSION' => $results->{mod_vers},
'CPAN_FILE' => $results->{download},
'CPAN_USERID' => $results->{cpanid},
where C<$results> are the results returned from C<CPAN::SQLite>.
=item dist
The attributes are
'DESCRIPTION' => $results->{dist_abs},
'CPAN_USERID' => $results->{cpanid},
'CPAN_VERSION' => $results->{dist_vers},
As well, a C<CONTAINSMODS> key to C<$CPAN::META> is added, this
being a hash reference whose keys are the modules contained
within the distribution.
=back
There is also a method available C<reload>, which rebuilds
the database. It can be used as
$obj->reload(force => 1, time => $time);
The C<time> option (which, if not passed in, will default to the
current time) will be used to compare the current time to
the mtime of the database file; if they differ by more than
one day, the database will be rebuilt. The <force> option, if
given, will force a rebuilding of the database regardless
of the time difference.
=cut

View File

@@ -0,0 +1,827 @@
# $Id: Populate.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::Populate;
use strict;
use warnings;
no warnings qw(redefine);
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use CPAN::SQLite::Util qw($table_id has_hash_data print_debug);
use CPAN::SQLite::DBI::Index;
use CPAN::SQLite::DBI qw($dbh);
use File::Find;
use File::Basename;
use File::Spec::Functions;
use File::Path;
use Scalar::Util 'weaken';
our $dbh = $CPAN::SQLite::DBI::dbh;
my ($setup);
my %tbl2obj;
$tbl2obj{$_} = __PACKAGE__ . '::' . $_ foreach (qw(dists mods auths info));
my %obj2tbl = reverse %tbl2obj;
sub new {
my ($class, %args) = @_;
$setup = $args{setup};
my $index = $args{index};
my @tables = qw(dists mods auths info);
foreach my $table (@tables) {
my $obj = $index->{$table};
die "Please supply a CPAN::SQLite::Index::$table object"
unless ($obj and ref($obj) eq "CPAN::SQLite::Index::$table");
}
my $state = $args{state};
unless ($setup) {
die "Please supply a CPAN::SQLite::State object"
unless ($state and ref($state) eq 'CPAN::SQLite::State');
}
my $cdbi = CPAN::SQLite::DBI::Index->new(%args);
my $self = {
index => $index,
state => $state,
obj => {},
cdbi => $cdbi,
db_name => $args{db_name},
};
return bless $self, $class;
}
sub populate {
my $self = shift;
if ($setup) {
unless ($self->{cdbi}->create_tables(setup => $setup)) {
warn "Creating tables failed";
return;
}
}
unless ($self->create_objs()) {
warn "Cannot create objects";
return;
}
unless ($self->populate_tables()) {
warn "Populating tables failed";
return;
}
return 1;
}
sub create_objs {
my $self = shift;
my @tables = qw(dists auths mods info);
foreach my $table (@tables) {
my $obj;
my $pack = $tbl2obj{$table};
my $index = $self->{index}->{$table};
if ($index and ref($index) eq "CPAN::SQLite::Index::$table") {
my $info = $index->{info};
if ($table ne 'info') {
return unless has_hash_data($info);
}
$obj = $pack->new(
info => $info,
cdbi => $self->{cdbi}->{objs}->{$table});
} else {
$obj = $pack->new(cdbi => $self->{cdbi}->{objs}->{$table});
}
$self->{obj}->{$table} = $obj;
}
foreach my $table (@tables) {
my $obj = $self->{obj}->{$table};
foreach (@tables) {
next if ref($obj) eq $tbl2obj{$_};
$obj->{obj}->{$_} = $self->{obj}->{$_};
weaken $obj->{obj}->{$_};
}
}
unless ($setup) {
my $state = $self->{state};
my @tables = qw(auths dists mods);
my @data = qw(ids insert update delete);
foreach my $table (@tables) {
my $state_obj = $state->{obj}->{$table};
my $pop_obj = $self->{obj}->{$table};
$pop_obj->{$_} = $state_obj->{$_} for (@data);
}
}
return 1;
}
sub populate_tables {
my $self = shift;
my @methods = $setup ? qw(insert) : qw(insert update delete);
# Reset status
my $info_obj = $self->{'obj'}->{'info'};
unless ($info_obj->delete) {
print_debug('Fatal error from ', ref($info_obj), ':', $info_obj->{'error_msg'});
return;
}
my @tables = qw(auths dists mods);
for my $method (@methods) {
for my $table (@tables) {
my $obj = $self->{obj}->{$table};
unless ($obj->$method()) {
if (my $error = $obj->{error_msg}) {
print_debug("Fatal error from ", ref($obj), ": ", $error, $/);
return;
} else {
my $info = $obj->{info_msg};
print_debug("Info from ", ref($obj), ": ", $info, $/);
}
}
}
}
# Update status
unless ($info_obj->insert) {
print_debug('Fatal error from ', ref($info_obj), ':', $info_obj->{'error_msg'});
return;
}
return 1;
}
package CPAN::SQLite::Populate::auths;
use parent 'CPAN::SQLite::Populate';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No author info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::auths');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub insert {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $info = $self->{info};
my $cdbi = $self->{cdbi};
my $data = $setup ? $info : $self->{insert};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No author data to insert};
return;
}
my $auth_ids = $self->{ids};
my @fields = qw(cpanid email fullname);
my $sth = $cdbi->sth_insert(\@fields) or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
foreach my $cpanid (keys %$data) {
my $values = $info->{$cpanid};
next unless ($values and $cpanid);
print_debug("Inserting author $cpanid\n");
$sth->execute($cpanid, $values->{email}, $values->{fullname})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$auth_ids->{$cpanid} = $dbh->func('last_insert_rowid') or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
}
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub update {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $data = $self->{update};
my $cdbi = $self->{cdbi};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No author data to update};
return;
}
my $info = $self->{info};
my @fields = qw(cpanid email fullname);
foreach my $cpanid (keys %$data) {
print_debug("Updating author $cpanid\n");
next unless $data->{$cpanid};
my $sth = $cdbi->sth_update(\@fields, $data->{$cpanid});
my $values = $info->{$cpanid};
next unless ($cpanid and $values);
$sth->execute($cpanid, $values->{email}, $values->{fullname})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->finish();
undef $sth;
}
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub delete {
my $self = shift;
$self->{info_msg} = q{No author data to delete};
return;
}
package CPAN::SQLite::Populate::dists;
use parent 'CPAN::SQLite::Populate';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No dist info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::dists');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub insert {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
return unless my $auth_obj = $self->{obj}->{auths};
my $cdbi = $self->{cdbi};
my $auth_ids = $auth_obj->{ids};
my $dists = $self->{info};
my $data = $setup ? $dists : $self->{insert};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No dist data to insert};
return;
}
unless ($dists and $auth_ids) {
$self->{error_msg}->{index} = q{No dist index data available};
return;
}
my $dist_ids = $self->{ids};
my @fields = qw(auth_id dist_name dist_file dist_vers dist_abs);
my $sth = $cdbi->sth_insert(\@fields) or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
foreach my $distname (keys %$data) {
my $values = $dists->{$distname};
my $cpanid = $values->{cpanid};
next unless ($values and $cpanid and $auth_ids->{$cpanid});
print_debug("Inserting $distname of $cpanid\n");
$sth->execute($auth_ids->{ $values->{cpanid} }, $distname, $values->{dist_file}, $values->{dist_vers}, $values->{dist_abs})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$dist_ids->{$distname} = $dbh->func('last_insert_rowid') or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
}
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub update {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $cdbi = $self->{cdbi};
my $data = $self->{update};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No dist data to update};
return;
}
return unless my $auth_obj = $self->{obj}->{auths};
my $auth_ids = $auth_obj->{ids};
my $dists = $self->{info};
unless ($dists and $auth_ids) {
$self->{error_msg} = q{No dist index data available};
return;
}
my @fields = qw(auth_id dist_name dist_file dist_vers dist_abs);
foreach my $distname (keys %$data) {
next unless $data->{$distname};
my $sth = $cdbi->sth_update(\@fields, $data->{$distname});
my $values = $dists->{$distname};
my $cpanid = $values->{cpanid};
next unless ($values and $cpanid and $auth_ids->{$cpanid});
print_debug("Updating $distname of $cpanid\n");
$sth->execute($auth_ids->{ $values->{cpanid} }, $distname, $values->{dist_file}, $values->{dist_vers}, $values->{dist_abs})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->finish();
undef $sth;
}
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub delete {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $cdbi = $self->{cdbi};
my $data = $self->{delete};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No dist data to delete};
return;
}
my $sth = $cdbi->sth_delete('dist_id');
foreach my $distname (keys %$data) {
print_debug("Deleting $distname\n");
$sth->execute($data->{$distname}) or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
}
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
package CPAN::SQLite::Populate::mods;
use parent 'CPAN::SQLite::Populate';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No module info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::mods');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub insert {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
return unless my $dist_obj = $self->{obj}->{dists};
my $cdbi = $self->{cdbi};
my $dist_ids = $dist_obj->{ids};
my $mods = $self->{info};
my $data = $setup ? $mods : $self->{insert};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No module data to insert};
return;
}
unless ($mods and $dist_ids) {
$self->{error_msg} = q{No module index data available};
return;
}
my $mod_ids = $self->{ids};
my @fields = qw(dist_id mod_name mod_abs
mod_vers);
my $sth = $cdbi->sth_insert(\@fields) or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
foreach my $modname (keys %$data) {
my $values = $mods->{$modname};
next unless ($values and $dist_ids->{ $values->{dist_name} });
$sth->execute($dist_ids->{ $values->{dist_name} }, $modname, $values->{mod_abs}, $values->{mod_vers})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$mod_ids->{$modname} = $dbh->func('last_insert_rowid') or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
}
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub update {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $cdbi = $self->{cdbi};
my $data = $self->{update};
unless (has_hash_data($data)) {
$self->{info_msg} = q{No module data to update};
return;
}
return unless my $dist_obj = $self->{obj}->{dists};
my $dist_ids = $dist_obj->{ids};
my $mods = $self->{info};
unless ($dist_ids and $mods) {
$self->{error_msg} = q{No module index data available};
return;
}
my @fields = qw(dist_id mod_name mod_abs
mod_vers);
foreach my $modname (keys %$data) {
next unless $data->{$modname};
print_debug("Updating $modname\n");
my $sth = $cdbi->sth_update(\@fields, $data->{$modname});
my $values = $mods->{$modname};
next unless ($values and $dist_ids->{ $values->{dist_name} });
$sth->execute($dist_ids->{ $values->{dist_name} }, $modname, $values->{mod_abs}, $values->{mod_vers})
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->finish();
undef $sth;
}
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub delete {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
return unless my $dist_obj = $self->{obj}->{dists};
my $cdbi = $self->{cdbi};
my $data = $dist_obj->{delete};
if (has_hash_data($data)) {
my $sth = $cdbi->sth_delete('dist_id');
foreach my $distname (keys %$data) {
$sth->execute($data->{$distname}) or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
}
$sth->finish();
undef $sth;
}
$data = $self->{delete};
if (has_hash_data($data)) {
my $sth = $cdbi->sth_delete('mod_id');
foreach my $modname (keys %$data) {
$sth->execute($data->{$modname}) or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
print_debug("Deleting $modname\n");
}
$sth->finish;
undef $sth;
}
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
package CPAN::SQLite::Populate::info;
use parent 'CPAN::SQLite::Populate';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::info');
my $self = {
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub insert {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $cdbi = $self->{cdbi};
my $sth = $cdbi->sth_insert(['status']) or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->execute(1)
or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub update {
my $self = shift;
$self->{'error_msg'} = 'update is not a valid call';
return;
}
sub delete {
my $self = shift;
unless ($dbh) {
$self->{error_msg} = q{No db handle available};
return;
}
my $cdbi = $self->{cdbi};
my $sth = $cdbi->sth_delete('status');
$sth->execute(1) or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
$sth->finish();
undef $sth;
$dbh->commit() or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
package CPAN::SQLite::Populate;
sub db_error {
my ($obj, $sth) = @_;
return unless $dbh;
if ($sth) {
$sth->finish;
undef $sth;
}
return $obj->{error_msg} = q{Database error: } . $dbh->errstr;
}
1;
=head1 NAME
CPAN::SQLite::Populate - create and populate database tables
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module is responsible for creating the tables
(if C<setup> is passed as an option) and then for
inserting, updating, or deleting (as appropriate) the
relevant information from the indices of
I<CPAN::SQLite::Info> and the
state information from I<CPAN::SQLite::State>. It does
this through the C<insert>, C<update>, and C<delete>
methods associated with each table.
Note that the tables are created with the C<setup> argument
passed into the C<new> method when creating the
C<CPAN::SQLite::Index> object; existing tables will be
dropped.
=head1 TABLES
The tables used are described below - the data types correspond
to mysql tables, with the corresponding adjustments made if
the SQLite database is used.
=head2 mods
This table contains module information, and is created as
mod_id INTEGER NOT NULL PRIMARY KEY
mod_name VARCHAR(100) NOT NULL
dist_id INTEGER NOT NULL
mod_abs TEXT
mod_vers VARCHAR(10)
=over 3
=item * mod_id
This is the primary (unique) key of the table.
=item * dist_id
This key corresponds to the id of the associated distribution
in the C<dists> table.
=item * mod_name
This is the module's name.
=item * mod_abs
This is a description, if available, of the module.
=item * mod_vers
This value, if present, gives the version of the module.
=back
=head2 dists
This table contains distribution information, and is created as
dist_id INTEGER NOT NULL PRIMARY KEY
dist_name VARCHAR(90) NOT NULL
auth_id INTEGER NOT NULL
dist_file VARCHAR(110) NOT NULL
dist_vers VARCHAR(20)
dist_abs TEXT
=over 3
=item * dist_id
This is the primary (unique) key of the table.
=item * auth_id
This corresponds to the CPAN author id of the distribution
in the C<auths> table.
=item * dist_name
This corresponds to the distribution name (eg, for
F<My-Distname-0.22.tar.gz>, C<dist_name> will be C<My-Distname>).
=item * dist_file
This corresponds to the CPAN file name.
=item * dist_vers
This is the version of the CPAN file (eg, for
F<My-Distname-0.22.tar.gz>, C<dist_vers> will be C<0.22>).
=item * dist_abs
This is a description of the distribution. If not directly
supplied, the description for, eg, C<Foo::Bar>, if present, will
be used for the C<Foo-Bar> distribution.
=back
=head2 auths
This table contains CPAN author information, and is created as
auth_id INTEGER NOT NULL PRIMARY KEY
cpanid VARCHAR(20) NOT NULL
fullname VARCHAR(40) NOT NULL
email TEXT
=over 3
=item * auth_id
This is the primary (unique) key of the table.
=item * cpanid
This gives the CPAN author id.
=item * fullname
This is the full name of the author.
=item * email
This is the supplied email address of the author.
=back
=head1 SEE ALSO
L<CPAN::SQLite::Index>
=cut

View File

@@ -0,0 +1,517 @@
# $Id: Search.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::Search;
use strict;
use warnings;
no warnings qw(redefine);
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use utf8;
use CPAN::SQLite::Util qw($mode_info);
use CPAN::SQLite::DBI::Search;
our $max_results = 0;
my $cdbi_query;
my %mode2obj;
$mode2obj{$_} = __PACKAGE__ . '::' . $_ for (qw(dist author module));
sub new {
my ($class, %args) = @_;
$cdbi_query = CPAN::SQLite::DBI::Search->new(%args);
$max_results = $args{max_results} if $args{max_results};
my $self = { results => undef, error => '', %args };
return bless $self, $class;
}
sub query {
my ($self, %args) = @_;
my $mode = $args{mode} || 'module';
unless ($mode) {
$self->{error} = q{Please specify a 'mode' argument};
return;
}
my $info = $mode_info->{$mode};
my $table = $info->{table};
unless ($table) {
$self->{error} = qq{No table exists for '$mode'};
return;
}
my $cdbi = $cdbi_query->{objs}->{$table};
my $class = 'CPAN::SQLite::DBI::Search::' . $table;
unless ($cdbi and ref($cdbi) eq $class) {
$self->{error} = qq{No cdbi object exists for '$table'};
return;
}
my $obj;
eval { $obj = $mode2obj{$mode}->make(table => $table, cdbi => $cdbi); };
if ($@) {
$self->{error} = qq{Mode '$mode' is not known};
return;
}
my $value;
my $search = {
name => $info->{name},
text => $info->{text},
id => $info->{id},
};
TYPE: {
($value = $args{query}) and do {
$search->{value} = $value;
$search->{type} = 'query';
$search->{wantarray} = 1;
last TYPE;
};
($value = $args{id}) and do {
$search->{value} = $value;
$search->{type} = 'id';
$search->{distinct} = 1;
last TYPE;
};
($value = $args{name}) and do {
$search->{value} = $value;
$search->{type} = 'name';
$search->{distinct} = 1;
last TYPE;
};
$self->{error} = q{Cannot determine the type of search};
return;
}
$obj->search(search => $search, meta_obj => $self->{meta_obj});
$self->{results} = $obj->{results};
if (my $error = $obj->{error}) {
$self->{error} = $error;
return;
}
return 1;
}
sub make {
my ($class, %args) = @_;
for (qw(table cdbi)) {
die qq{Must supply an '$_' arg} unless defined $args{$_};
}
my $self = {
results => undef,
error => '',
table => $args{table},
cdbi => $args{cdbi} };
return bless $self, $class;
}
package CPAN::SQLite::Search::author;
use parent 'CPAN::SQLite::Search';
sub search {
my ($self, %args) = @_;
return unless $args{search};
my $cdbi = $self->{cdbi};
my $meta_obj = $args{meta_obj};
$args{fields} = [qw(auth_id cpanid fullname email)];
$args{table} = 'auths';
if ($max_results) {
$args{limit} = $max_results;
}
$args{order_by} = 'cpanid';
my $results;
return unless $results = (
$meta_obj
? $cdbi->fetch_and_set(%args)
: $cdbi->fetch(%args));
unless ($meta_obj) {
$self->{results} = (ref($results) eq 'ARRAY' and scalar @$results == 1) ? $results->[0] : $results;
}
return 1 if $meta_obj;
# The following will get all the dists associated with the cpanid
$args{join} = undef;
$args{table} = 'dists';
$args{fields} = [qw(dist_file dist_abs)];
$args{order_by} = 'dist_file';
my @items = (ref($results) eq 'ARRAY') ? @$results : ($results);
foreach my $item (@items) {
my $search = {
id => 'auth_id',
value => $item->{auth_id},
type => 'id',
wantarray => 1,
};
my $dists;
next unless ($dists = $cdbi->fetch(%args, search => $search));
$item->{dists} = (ref($dists) eq 'ARRAY') ? $dists : [$dists];
}
$self->{results} = (ref($results) eq 'ARRAY' and scalar @$results == 1) ? $results->[0] : $results;
return 1;
}
package CPAN::SQLite::Search::module;
use parent 'CPAN::SQLite::Search';
sub search {
my ($self, %args) = @_;
return unless $args{search};
my $cdbi = $self->{cdbi};
my $meta_obj = $args{meta_obj};
$args{fields} = [
qw(mod_id mod_name mod_abs mod_vers
dist_id dist_name dist_file dist_vers dist_abs
auth_id cpanid fullname email)
];
$args{table} = 'dists';
$args{join} = {
mods => 'dist_id',
auths => 'auth_id',
};
$args{order_by} = 'mod_name';
if ($max_results) {
$args{limit} = $max_results;
}
my $results;
return unless $results = (
$meta_obj
? $cdbi->fetch_and_set(%args, want_ids => 1)
: $cdbi->fetch(%args));
# if running under CPAN.pm, need to build a list of modules
# contained in the distribution
if ($meta_obj) {
my %seen;
$args{join} = undef;
$args{table} = 'mods';
my @items = (ref($results) eq 'ARRAY') ? @$results : ($results);
foreach my $item (@items) {
my $dist_id = $item->{dist_id};
next if $seen{$dist_id};
$args{fields} = [qw(mod_name mod_abs)];
$args{order_by} = 'mod_name';
$args{join} = undef;
my $search = {
id => 'dist_id',
value => $item->{dist_id},
type => 'id',
wantarray => 1,
};
$seen{$dist_id}++;
my $mods;
next unless $mods = $cdbi->fetch_and_set(
%args,
search => $search,
set_list => 1,
download => $item->{download});
}
}
unless ($meta_obj) {
$self->{results} = (ref($results) eq 'ARRAY' and scalar @$results == 1) ? $results->[0] : $results;
}
return 1;
}
package CPAN::SQLite::Search::dist;
use parent 'CPAN::SQLite::Search';
sub search {
my ($self, %args) = @_;
return unless $args{search};
my $cdbi = $self->{cdbi};
my $meta_obj = $args{meta_obj};
$args{fields} = [
qw(dist_id dist_name dist_abs dist_vers
dist_file auth_id cpanid fullname email)
];
$args{table} = 'dists';
$args{join} = { auths => 'auth_id' };
$args{order_by} = 'dist_name';
if ($max_results) {
$args{limit} = $max_results;
}
my $results;
return unless $results = (
$meta_obj
? $cdbi->fetch_and_set(%args, want_ids => 1)
: $cdbi->fetch(%args));
$args{join} = undef;
$args{table} = 'mods';
$args{fields} = [qw(mod_name mod_abs)];
$args{order_by} = 'mod_name';
my @items = (ref($results) eq 'ARRAY') ? @$results : ($results);
foreach my $item (@items) {
my $search = {
id => 'dist_id',
value => $item->{dist_id},
type => 'id',
wantarray => 1,
};
my $mods;
next
unless $mods = (
$meta_obj
? $cdbi->fetch_and_set(
%args,
search => $search,
set_list => 1,
download => $item->{download})
: $cdbi->fetch(%args, search => $search));
next if $meta_obj;
$item->{mods} = (ref($mods) eq 'ARRAY') ? $mods : [$mods];
}
unless ($meta_obj) {
$self->{results} = (ref($results) eq 'ARRAY' and scalar @$results == 1) ? $results->[0] : $results;
}
return 1;
}
1;
=head1 NAME
CPAN::SQLite::Search - perform queries on the database
=head1 VERSION
version 0.219
=head1 SYNOPSIS
my $max_results = 200;
my $query = CPAN::SQLite::Search->new(db_dir => $db_dir,
db_name => $db_name,
max_results => $max_results);
$query->query(mode => 'module', name => 'Net::FTP');
my $results = $query->{results};
=head1 CONSTRUCTING THE QUERY
This module queries the database via various types of queries
and returns the results for subsequent display. The
C<CPAN::SQLite::Search> object is created via the C<new> method as
my $query = CPAN::SQLite::Search->new(db_dir => $db_dir,
db_name => $db_name,
max_results => $max_results);
which takes as arguments
=over 3
=item * db_dir =E<gt> $db_dir
This is the directory where the database file is stored. This is
optional if the C<CPAN> option is given.
=item * CPAN =E<gt> $CPAN
This option specifies the C<cpan_home> directory of an
already configured CPAN.pm, which is where the database
file will be stored if C<db_dir> is not given.
=item * max_results =E<gt> $max_results
This is the maximum value used to limit the number of results
returned under a user query. If not specified, a value contained
within C<CPAN::SQLite::Search> will be used.
=back
A basic query then is constructed as
$query->query(mode => $mode, $type => $value);
with the results available as
my $results = $query->{results}
There are three basic modes:
=over 3
=item * module
This is for information on modules.
=item * dist
This is for information on distributions.
=item * author
This is for information on CPAN authors or cpanids.
=back
=head2 C<module>, C<dist>, and C<author> modes
For a mode of C<module>, C<dist>, and C<author>, there are
four basic options to be used for the C<$type =E<gt> $value> option:
=over 3
=item * query =E<gt> $query_term
This will search through module names,
distribution names, or CPAN author names and ids
(for C<module>, C<dist>, and C<author> modes
respectively). The results are case insensitive,
and Perl regular expressions for the C<$query_term>
are recognized.
=item * name =E<gt> $name
This will report exact matches (in a case sensitive manner)
for the module name, distribution name, or CPAN author id,
for C<module>, C<dist>, and C<author> modes
respectively.
=item * id =E<gt> $id
This will look up information on the primary key according
to the mode specified. This is more for internal use,
to help speed up queries; using this "publically" is
probably not a good idea, as the ids may change over the
course of time.
=back
=head1 RESULTS
After making the query, the results can be accessed through
my $results = $query->{results};
No results either can mean no matches were found, or
else an error in making the query resulted (in which case,
a brief error message is contained in C<$query-E<gt>{error}>).
Assuming there are results, what is returned depends on
the mode and on the type of query. See L<CPAN::SQLite::Populate>
for a description of the fields in the various tables
listed below - these fields are used as the keys of the
hash references that arise.
=head2 C<author> mode
=over 3
=item * C<name> or C<id> query
This returns the C<auth_id>, C<cpanid>, C<email>, and C<fullname>
of the C<auths> table. As well, an array reference
C<$results-E<gt>{dists}> is returned representing
all distributions associated with that C<cpanid> - each
member of the array reference is a hash reference
describing the C<dist_id>, C<dist_name>,
C<dist_abs>, C<dist_vers>, and C<dist_file> fields in the
C<dists> table. An additional entry, C<download>, is
supplied, which can be used as C<$CPAN/authors/id/$download>
to specify the url of the distribution.
=item * C<query> query
If this results in more than one match, an array reference
is returned, each member of which is a hash reference containing
the C<auth_id>, C<cpanid>, and C<fullname> fields. If there
is only one result found, a C<name> query based on the
matched C<cpanid> is performed.
=back
=head2 C<module> mode
=over 3
=item * C<name> or C<id> query
This returns the C<mod_id>, C<mod_name>, C<mod_abs>, C<mod_vers>,
C<dist_id>, C<dist_name>, C<dist_file>,
C<auth_id>, C<cpanid>, C<fullname>, and C<email>
of the C<auths>, C<mods>, and C<dists> tables.
As well, the following entries may be present.
=over 3
=item * C<download>
This can be used as C<$CPAN/authors/id/$download>
to specify the url of the distribution.
=back
=item * C<query> query
If this results in more than one match, an array reference
is returned, each member of which is a hash reference containing
the C<mod_id>, C<mod_name>, C<mod_abs>, C<mod_abs>, C<dist_vers>, C<dist_abs>,
C<auth_id>, C<cpanid>, C<dist_id>, C<dist_name>, and C<dist_file>.
As well, a C<download> field which
can be used as C<$CPAN/authors/id/$download>
to specify the url of the distribution is provided. If there
is only one result found, a C<name> query based on the
matched C<mod_name> is performed.
=back
=head2 C<dist> mode
=over 3
=item * C<name> or C<id> query
This returns the C<dist_id>, C<dist_name>, C<dist_abs>, C<dist_vers>,
C<dist_file>, C<size>, C<birth>, C<auth_id>, C<cpanid>, and C<fullname>
of the C<auths>, C<mods>, and C<dists> tables.
As well, the following entries may be present.
=over 3
=item * C<download>
This can be used as C<$CPAN/authors/id/$download>
to specify the url of the distribution.
=item * C<mods>
This is an array reference containing information on the
modules present. Each entry is a hash reference containing the
C<mod_id>, C<mod_name>, C<mod_abs>, and C<mod_vers>
fields for the module.
=back
=item * C<query> query
If this results in more than one match, an array reference
is returned, each member of which is a hash reference containing
the C<dist_id>, C<dist_name>, C<dist_abs>, C<dist_file>,
and C<cpanid> fields. As well, a C<download> field which
can be used as C<$CPAN/authors/id/$download>
to specify the url of the distribution is provided. If there
is only one result found, a C<name> query based on the
matched C<dist_name> is performed.
=back
=head1 SEE ALSO
L<CPAN::SQLite::Populate>.
=head1 AUTHORS
Randy Kobes (passed away on September 18, 2010)
Serguei Trouchelle E<lt>stro@cpan.orgE<gt>
=head1 COPYRIGHT
Copyright 2006,2008 by Randy Kobes E<lt>r.kobes@uwinnipeg.caE<gt>.
Copyright 2011-2013 by Serguei Trouchelle E<lt>stro@cpan.orgE<gt>.
Use and redistribution are under the same terms as Perl itself.
=cut

View File

@@ -0,0 +1,452 @@
# $Id: State.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::State;
use strict;
use warnings;
no warnings qw(redefine);
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use CPAN::SQLite::DBI qw($dbh);
use CPAN::SQLite::DBI::Index;
use CPAN::SQLite::Util qw(has_hash_data print_debug);
use Scalar::Util 'weaken';
my %tbl2obj;
$tbl2obj{$_} = __PACKAGE__ . '::' . $_ for (qw(dists mods auths info));
my %obj2tbl = reverse %tbl2obj;
our $dbh = $CPAN::SQLite::DBI::dbh;
sub new {
my ($class, %args) = @_;
if ($args{setup}) {
die "No state information available under setup";
}
my $index = $args{index};
my @tables = qw(dists mods auths info);
foreach my $table (@tables) {
my $obj = $index->{$table};
die "Please supply a CPAN::SQLite::Index::$table object"
unless ($obj and ref($obj) eq "CPAN::SQLite::Index::$table");
}
my $cdbi = CPAN::SQLite::DBI::Index->new(%args);
my $self = {
index => $index,
obj => {},
cdbi => $cdbi,
reindex => $args{reindex},
};
return bless $self, $class;
}
sub state {
my $self = shift;
unless ($self->create_objs()) {
print_debug("Cannot create objects");
return;
}
unless ($self->state_info()) {
print_debug("Getting state information failed");
return;
}
return 1;
}
sub create_objs {
my $self = shift;
my @tables = qw(dists auths mods info);
foreach my $table (@tables) {
my $obj;
my $pack = $tbl2obj{$table};
my $index = $self->{index}->{$table};
if ($index and ref($index) eq "CPAN::SQLite::Index::$table") {
my $info = $index->{info};
if ($table ne 'info') {
return unless has_hash_data($info);
}
$obj = $pack->new(
info => $info,
cdbi => $self->{cdbi}->{objs}->{$table});
} else {
$obj = $pack->new();
}
$self->{obj}->{$table} = $obj;
}
foreach my $table (@tables) {
my $obj = $self->{obj}->{$table};
foreach (@tables) {
next if ref($obj) eq $tbl2obj{$_};
$obj->{obj}->{$_} = $self->{obj}->{$_};
weaken $obj->{obj}->{$_};
}
}
return 1;
}
sub state_info {
my $self = shift;
my @methods = qw(ids state);
my @tables = qw(dists auths mods);
for my $method (@methods) {
for my $table (@tables) {
my $obj = $self->{obj}->{$table};
unless ($obj->$method()) {
if (my $error = $obj->{error_msg}) {
print_debug("Fatal error from ", ref($obj), ": ", $error, $/);
return;
} else {
my $info = $obj->{info_msg};
print_debug("Info from ", ref($obj), ": ", $info, $/);
}
}
}
}
# Check "info"
if (my $obj = $self->{'obj'}->{'info'}) {
return unless $obj->state;
}
return 1;
}
package CPAN::SQLite::State::auths;
use parent 'CPAN::SQLite::State';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No author info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::auths');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub ids {
my $self = shift;
my $cdbi = $self->{cdbi};
$self->{ids} = $cdbi->fetch_ids() or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub state {
my $self = shift;
my $auth_ids = $self->{ids};
return unless my $dist_obj = $self->{obj}->{dists};
my $dist_update = $dist_obj->{update};
my $dist_insert = $dist_obj->{insert};
my $dists = $dist_obj->{info};
my ($update, $insert);
if (has_hash_data($dist_insert)) {
foreach my $distname (keys %{$dist_insert}) {
my $cpanid = $dists->{$distname}->{cpanid};
if (my $auth_id = $auth_ids->{$cpanid}) {
$update->{$cpanid} = $auth_id;
} else {
$insert->{$cpanid}++;
}
}
}
if (has_hash_data($dist_update)) {
foreach my $distname (keys %{$dist_update}) {
my $cpanid = $dists->{$distname}->{cpanid};
if (my $auth_id = $auth_ids->{$cpanid}) {
$update->{$cpanid} = $auth_id;
} else {
$insert->{$cpanid}++;
}
}
}
$self->{update} = $update;
$self->{insert} = $insert;
return 1;
}
package CPAN::SQLite::State::dists;
use parent 'CPAN::SQLite::State';
use CPAN::SQLite::Util qw(vcmp has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No dist info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::dists');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
versions => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
reindex => undef,
};
return bless $self, $class;
}
sub ids {
my $self = shift;
my $cdbi = $self->{cdbi};
($self->{ids}, $self->{versions}) = $cdbi->fetch_ids() or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub state {
my $self = shift;
my $dist_versions = $self->{versions};
my $dists = $self->{info};
my $dist_ids = $self->{ids};
my ($insert, $update, $delete);
my $reindex = $self->{reindex};
if (defined $reindex) {
my @dists = ref($reindex) eq 'ARRAY' ? @$reindex : ($reindex);
foreach my $distname (@dists) {
my $id = $dist_ids->{$distname};
if (not defined $id) {
print_debug(qq{"$distname" does not have an id: reindexing ignored\n});
next;
}
$update->{$distname} = $id;
}
$self->{update} = $update;
return 1;
}
foreach my $distname (keys %$dists) {
if (not defined $dist_versions->{$distname}) {
$insert->{$distname}++;
} elsif (vcmp($dists->{$distname}->{dist_vers}, $dist_versions->{$distname}) > 0) {
$update->{$distname} = $dist_ids->{$distname};
}
}
$self->{update} = $update;
$self->{insert} = $insert;
foreach my $distname (keys %$dist_versions) {
next if $dists->{$distname};
$delete->{$distname} = $dist_ids->{$distname};
print_debug("Will delete $distname\n");
}
$self->{delete} = $delete;
return 1;
}
package CPAN::SQLite::State::mods;
use parent 'CPAN::SQLite::State';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $info = $args{info};
die "No module info available" unless has_hash_data($info);
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::mods');
my $self = {
info => $info,
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub ids {
my $self = shift;
my $cdbi = $self->{cdbi};
$self->{ids} = $cdbi->fetch_ids() or do {
$self->{error_msg} = $cdbi->{error_msg};
return;
};
return 1;
}
sub state {
my $self = shift;
my $mod_ids = $self->{ids};
return unless my $dist_obj = $self->{obj}->{dists};
my $dists = $dist_obj->{info};
my $dist_update = $dist_obj->{update};
my $dist_insert = $dist_obj->{insert};
my ($update, $insert, $delete);
my $cdbi = $self->{cdbi};
if (has_hash_data($dist_insert)) {
foreach my $distname (keys %{$dist_insert}) {
foreach my $module (keys %{ $dists->{$distname}->{modules} }) {
$insert->{$module}++;
}
}
}
if (has_hash_data($dist_update)) {
foreach my $distname (keys %{$dist_update}) {
foreach my $module (keys %{ $dists->{$distname}->{modules} }) {
my $mod_id = $mod_ids->{$module};
if ($mod_id) {
$update->{$module} = $mod_id;
} else {
$insert->{$module}++;
}
}
}
}
if (has_hash_data($dist_update)) {
my $sql = q{SELECT mod_id,mod_name from mods,dists WHERE dists.dist_id = mods.dist_id and dists.dist_id = ?};
my $sth = $dbh->prepare($sql) or do {
$cdbi->db_error();
$self->{error_msg} = $cdbi->{error_msg};
return;
};
my $dist_ids = $dist_obj->{ids};
foreach my $distname (keys %{$dist_update}) {
my %mods = ();
%mods = map { $_ => 1 } keys %{ $dists->{$distname}->{modules} };
$sth->execute($dist_ids->{$distname}) or do {
$cdbi->db_error($sth);
$self->{error_msg} = $cdbi->{error_msg};
return;
};
while (my ($mod_id, $mod_name) = $sth->fetchrow_array) {
next if $mods{$mod_name};
$delete->{$mod_name} = $mod_id;
}
}
$sth->finish;
undef $sth;
}
$self->{update} = $update;
$self->{insert} = $insert;
$self->{delete} = $delete;
return 1;
}
package CPAN::SQLite::State::info;
use parent 'CPAN::SQLite::State';
use CPAN::SQLite::Util qw(has_hash_data print_debug);
sub new {
my ($class, %args) = @_;
my $cdbi = $args{cdbi};
die "No dbi object available"
unless ($cdbi and ref($cdbi) eq 'CPAN::SQLite::DBI::Index::info');
my $self = {
info => '',
insert => {},
update => {},
delete => {},
ids => {},
obj => {},
cdbi => $cdbi,
error_msg => '',
info_msg => '',
};
return bless $self, $class;
}
sub state {
my $self = shift;
return 1;
}
package CPAN::SQLite::State;
1;
__END__
=head1 NAME
CPAN::SQLite::State - get state information on the database
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module gets information on the current state of the
database and compares it to that obtained from the CPAN
index files from I<CPAN::SQLite::Info> and from the
repositories from I<CPAN::SQLite::PPM>. For each of the
four tables I<dists>, I<mods>, I<auths>, and I<ppms>,
two methods are used to get this information:
=over 3
=item * C<ids>
This method gets the ids of the relevant names, and
versions, if applicable, in the table.
=item * C<state>
This method compares the information in the tables
obtained from the C<ids> method to that from the
CPAN indices and ppm repositories. One of three actions
is then decided, which is subsequently acted upon in
I<CPAN::SQLite::Populate>.
=over 3
=item * C<insert>
If the information in the indices is not in the
database, this information is marked for insertion.
=item * C<update>
If the information in the database is older than that
form the indices (generally, this means an older version),
the information is marked for updating.
=item * C<delete>
If the information in the database is no longer present
in the indices, the information is marked for deletion.
=back
=back
=cut

View File

@@ -0,0 +1,265 @@
# $Id: Util.pm 84 2020-05-31 06:29:34Z stro $
package CPAN::SQLite::Util;
use strict;
use warnings;
our $VERSION = '0.219';
use English qw/-no_match_vars/;
use parent 'Exporter';
our (@EXPORT_OK, %modes, $table_id, $query_info, $mode_info, $full_id);
@EXPORT_OK = qw($repositories %modes
vcmp $table_id $query_info $mode_info $full_id
has_hash_data has_array_data
download print_debug);
make_ids();
$mode_info = {
module => {
id => 'mod_id',
table => 'mods',
name => 'mod_name',
text => 'mod_abs',
},
dist => {
id => 'dist_id',
table => 'dists',
name => 'dist_name',
text => 'dist_abs',
},
author => {
id => 'auth_id',
table => 'auths',
name => 'cpanid',
text => 'fullname',
},
};
%modes = map { $_ => 1 } keys %$mode_info;
$query_info = {
module => { mode => 'module', type => 'name' },
mod_id => { mode => 'module', type => 'id' },
dist => { mode => 'dist', type => 'name' },
dist_id => { mode => 'dist', type => 'id' },
cpanid => { mode => 'author', type => 'name' },
author => { mode => 'author', type => 'name' },
auth_id => { mode => 'author', type => 'id' },
};
sub make_ids {
my @tables = qw(mods dists auths);
foreach my $table (@tables) {
(my $id = $table) =~ s!(\w+)s$!$1_id!;
$table_id->{$table} = $id;
$full_id->{$id} = $table . '.' . $id;
}
return;
}
#my $num_re = qr{^0*\.\d+$};
#sub vcmp {
# my ($v1, $v2) = @_;
# return unless (defined $v1 and defined $v2);
# if ($v1 =~ /$num_re/ and $v2 =~ /$num_re/) {
# return $v1 <=> $v2;
# }
# return Sort::Versions::versioncmp($v1, $v2);
#}
sub has_hash_data {
my $data = shift;
return unless (defined $data and ref($data) eq 'HASH');
return (scalar keys %$data > 0) ? 1 : 0;
}
sub has_array_data {
my $data = shift;
return unless (defined $data and ref($data) eq 'ARRAY');
return (scalar @$data > 0) ? 1 : 0;
}
sub download {
my ($cpanid, $dist_file) = @_;
return unless ($cpanid and $dist_file);
(my $fullid = $cpanid) =~ s!^(\w)(\w)(.*)!$1/$1$2/$1$2$3!;
my $download = $fullid . '/' . $dist_file;
return $download;
}
sub print_debug {
return unless $ENV{CPAN_SQLITE_DEBUG};
$CPAN::FrontEnd->myprint(@_);
}
sub vcmp {
my ($v1, $v2) = @_;
return CPAN::SQLite::Version->vcmp($v1, $v2);
}
# This is borrowed essentially verbatim from CPAN::Version
# It's included here so as to not demand a CPAN.pm upgrade
package CPAN::SQLite::Version;
use strict;
our $VERSION = '0.219';
no warnings;
# CPAN::Version::vcmp courtesy Jost Krieger
sub vcmp {
my ($self, $l, $r) = @_;
return 0 if $l eq $r; # short circuit for quicker success
for ($l, $r) {
next unless tr/.// > 1;
s/^v?/v/;
1 while s/\.0+(\d)/.$1/;
}
if ($l =~ /^v/ <=> $r =~ /^v/) {
for ($l, $r) {
next if /^v/;
$_ = $self->float2vv($_);
}
}
return (
($l ne "undef") <=> ($r ne "undef")
|| ($] >= 5.006
&& $l =~ /^v/
&& $r =~ /^v/
&& $self->vstring($l) cmp $self->vstring($r))
|| $l <=> $r
|| $l cmp $r
);
}
sub vgt {
my ($self, $l, $r) = @_;
return $self->vcmp($l, $r) > 0;
}
sub vlt {
my ($self, $l, $r) = @_;
return 0 + ($self->vcmp($l, $r) < 0);
}
sub vstring {
my ($self, $n) = @_;
$n =~ s/^v//
or die "CPAN::Search::Lite::Version::vstring() called with invalid arg [$n]";
{
no warnings;
return pack "U*", split /\./, $n;
}
}
# vv => visible vstring
sub float2vv {
my ($self, $n) = @_;
my ($rev) = int($n);
$rev ||= 0;
my ($mantissa) = $n =~ /\.(\d{1,12})/; # limit to 12 digits to limit
# architecture influence
$mantissa ||= 0;
$mantissa .= "0" while length($mantissa) % 3;
my $ret = "v" . $rev;
while ($mantissa) {
$mantissa =~ s/(\d{1,3})//
or die "Panic: length>0 but not a digit? mantissa[$mantissa]";
$ret .= "." . int($1);
}
# warn "n[$n]ret[$ret]";
return $ret;
}
sub readable {
my ($self, $n) = @_;
$n =~ /^([\w\-\+\.]+)/;
return $1 if defined $1 && length($1) > 0;
# if the first user reaches version v43, he will be treated as "+".
# We'll have to decide about a new rule here then, depending on what
# will be the prevailing versioning behavior then.
if ($] < 5.006) { # or whenever v-strings were introduced
# we get them wrong anyway, whatever we do, because 5.005 will
# have already interpreted 0.2.4 to be "0.24". So even if he
# indexer sends us something like "v0.2.4" we compare wrongly.
# And if they say v1.2, then the old perl takes it as "v12"
warn("Suspicious version string seen [$n]\n");
return $n;
}
my $better = sprintf "v%vd", $n;
return $better;
}
1;
__END__
=head1 NAME
CPAN::SQLite::Util - export some common data structures used by CPAN::SQLite::*
=head1 VERSION
version 0.219
=head1 DESCRIPTION
This module exports some common data structures used by other
I<CPAN::Search::Lite::*> modules. At present these are
=over 3
=item * C<$table_id>
This is a hash reference whose keys are the tables used
and whose values are the associated primary keys.
=item * C<$full_id>
This is a hash reference whose keys are the primary keys
of the tables and whose values are the associated fully qualified
primary keys (ie, with the table name prepended).
=item * C<$mode_info>
This is a hash reference whose keys are the allowed
modes of I<CPAN::Search::Lite::Query> and whose associated values
are hash references with keys C<id>, C<name>, and C<text> describing
what columns to use for that key.
=item * C<$query_info>
This is a hash reference whose purpose is to provide
shortcuts to making queries using I<CPAN::Search::Lite::Query>. The
keys of this reference is the shortcut name, and the associated
value is a hash reference specifying the required I<mode> and
I<type> keys.
=item * C<vcmp>
This routine, used as
if (vcmp($v1, $v2) > 0) {
print "$v1 is higher than $v2\n";
}
is used to compare two versions, and returns 1/0/-1 if
the first argument is considered higher/equal/lower than
the second. It uses C<Sort::Versions>.
=back
=cut

View File

@@ -0,0 +1,3 @@
class: CPAN::Mini::Portable
skip_perl: 1
no_conn_cache: 1