Initial Commit

This commit is contained in:
Riley Schneider
2025-12-03 16:38:10 +01:00
parent c5e26bf594
commit b732d8d4b5
17680 changed files with 5977495 additions and 2 deletions

View File

@@ -0,0 +1,380 @@
package CPANPLUS::Internals::Source::Memory;
use base 'CPANPLUS::Internals::Source';
use strict;
use CPANPLUS::Error;
use CPANPLUS::Module;
use CPANPLUS::Module::Fake;
use CPANPLUS::Module::Author;
use CPANPLUS::Internals::Constants;
use File::Fetch;
use Archive::Extract;
use IPC::Cmd qw[can_run];
use File::Temp qw[tempdir];
use File::Basename qw[dirname];
use Params::Check qw[allow check];
use Module::Load::Conditional qw[can_load];
use Locale::Maketext::Simple Class => 'CPANPLUS', Style => 'gettext';
use vars qw[$VERSION];
$VERSION = "0.9910";
$Params::Check::VERBOSE = 1;
=head1 NAME
CPANPLUS::Internals::Source::Memory - In memory implementation
=cut
### flag to show if init_trees got its' data from storable. This allows
### us to not write an existing stored file back to disk
{ my $from_storable;
sub _init_trees {
my $self = shift;
my $conf = $self->configure_object;
my %hash = @_;
my($path,$uptodate,$verbose,$use_stored);
my $tmpl = {
path => { default => $conf->get_conf('base'), store => \$path },
verbose => { default => $conf->get_conf('verbose'), store => \$verbose },
uptodate => { required => 1, store => \$uptodate },
use_stored => { default => 1, store => \$use_stored },
};
check( $tmpl, \%hash ) or return;
### retrieve the stored source files ###
my $stored = $self->__memory_retrieve_source(
path => $path,
uptodate => $uptodate && $use_stored,
verbose => $verbose,
) || {};
### we got this from storable if $stored has keys..
$from_storable = keys %$stored ? 1 : 0;
### set up the trees
$self->_atree( $stored->{_atree} || {} );
$self->_mtree( $stored->{_mtree} || {} );
return 1;
}
sub _standard_trees_completed { return $from_storable }
sub _custom_trees_completed { return $from_storable }
sub _finalize_trees {
my $self = shift;
my $conf = $self->configure_object;
my %hash = @_;
my($path,$uptodate,$verbose);
my $tmpl = {
path => { default => $conf->get_conf('base'), store => \$path },
verbose => { default => $conf->get_conf('verbose'), store => \$verbose },
uptodate => { required => 1, store => \$uptodate },
};
{ local $Params::Check::ALLOW_UNKNOWN = 1;
check( $tmpl, \%hash ) or return;
}
### write the stored files to disk, so we can keep using them
### from now on, till they become invalid
### write them if the original sources weren't uptodate, or
### we didn't just load storable files
$self->__memory_save_source() if !$uptodate or not $from_storable;
return 1;
}
### saves current memory state
sub _save_state {
my $self = shift;
return $self->_finalize_trees( @_, uptodate => 0 );
}
}
sub _add_author_object {
my $self = shift;
my %hash = @_;
my $class;
my $tmpl = {
class => { default => 'CPANPLUS::Module::Author', store => \$class },
map { $_ => { required => 1 } }
qw[ author cpanid email ]
};
my $href = do {
local $Params::Check::NO_DUPLICATES = 1;
check( $tmpl, \%hash ) or return;
};
my $obj = $class->new( %$href, _id => $self->_id );
$self->author_tree->{ $href->{'cpanid'} } = $obj or return;
return $obj;
}
{
my $tmpl = {
class => { default => 'CPANPLUS::Module' },
map { $_ => { required => 1 } } qw[
module version path comment author package description dslip mtime
],
};
sub _add_module_object {
my $self = shift;
my %hash = @_;
my $href = do {
local $Params::Check::SANITY_CHECK_TEMPLATE = 0;
check( $tmpl, \%hash ) or return;
};
my $class = delete $href->{class};
my $obj = $class->new( %$href, _id => $self->_id );
### Every module get's stored as a module object ###
$self->module_tree->{ $href->{module} } = $obj or return;
return $obj;
}
}
{ my %map = (
_source_search_module_tree => [ module_tree => 'CPANPLUS::Module' ],
_source_search_author_tree => [ author_tree => 'CPANPLUS::Module::Author' ],
);
while( my($sub, $aref) = each %map ) {
no strict 'refs';
my($meth, $class) = @$aref;
*$sub = sub {
my $self = shift;
my $conf = $self->configure_object;
my %hash = @_;
my($authors,$list,$verbose,$type);
my $tmpl = {
data => { default => [],
strict_type=> 1, store => \$authors },
allow => { required => 1, default => [ ], strict_type => 1,
store => \$list },
verbose => { default => $conf->get_conf('verbose'),
store => \$verbose },
type => { required => 1, allow => [$class->accessors()],
store => \$type },
};
my $args = check( $tmpl, \%hash ) or return;
my @rv;
for my $obj ( values %{ $self->$meth } ) {
#push @rv, $auth if check(
# { $type => { allow => $list } },
# { $type => $auth->$type }
# );
push @rv, $obj if allow( $obj->$type() => $list );
}
return @rv;
}
}
}
=pod
=head2 $cb->__memory_retrieve_source(name => $name, [path => $path, uptodate => BOOL, verbose => BOOL])
This method retrieves a I<storable>d tree identified by C<$name>.
It takes the following arguments:
=over 4
=item name
The internal name for the source file to retrieve.
=item uptodate
A flag indicating whether the file-cache is up-to-date or not.
=item path
The absolute path to the directory holding the source files.
=item verbose
A boolean flag indicating whether or not to be verbose.
=back
Will get information from the config file by default.
Returns a tree on success, false on failure.
=cut
sub __memory_retrieve_source {
my $self = shift;
my %hash = @_;
my $conf = $self->configure_object;
my $tmpl = {
path => { default => $conf->get_conf('base') },
verbose => { default => $conf->get_conf('verbose') },
uptodate => { default => 0 },
};
my $args = check( $tmpl, \%hash ) or return;
### check if we can retrieve a frozen data structure with storable ###
my $storable = can_load( modules => {'Storable' => '0.0'} )
if $conf->get_conf('storable');
return unless $storable;
### $stored is the name of the frozen data structure ###
my $stored = $self->__memory_storable_file( $args->{path} );
if ($storable && -e $stored && -s _ && $args->{'uptodate'}) {
msg( loc("Retrieving %1", $stored), $args->{'verbose'} );
my $href = Storable::retrieve($stored);
return $href;
} else {
return;
}
}
=pod
=head2 $cb->__memory_save_source([verbose => BOOL, path => $path])
This method saves all the parsed trees in I<storable>d format if
C<Storable> is available.
It takes the following arguments:
=over 4
=item path
The absolute path to the directory holding the source files.
=item verbose
A boolean flag indicating whether or not to be verbose.
=back
Will get information from the config file by default.
Returns true on success, false on failure.
=cut
sub __memory_save_source {
my $self = shift;
my %hash = @_;
my $conf = $self->configure_object;
my $tmpl = {
path => { default => $conf->get_conf('base'), allow => DIR_EXISTS },
verbose => { default => $conf->get_conf('verbose') },
force => { default => 1 },
};
my $args = check( $tmpl, \%hash ) or return;
my $aref = [qw[_mtree _atree]];
### check if we can retrieve a frozen data structure with storable ###
my $storable;
$storable = can_load( modules => {'Storable' => '0.0'} )
if $conf->get_conf('storable');
return unless $storable;
my $to_write = {};
foreach my $key ( @$aref ) {
next unless ref( $self->$key );
$to_write->{$key} = $self->$key;
}
return unless keys %$to_write;
### $stored is the name of the frozen data structure ###
my $stored = $self->__memory_storable_file( $args->{path} );
if (-e $stored && not -w $stored) {
msg( loc("%1 not writable; skipped.", $stored), $args->{'verbose'} );
return;
}
msg( loc("Writing compiled source information to disk. This might take a little while."),
$args->{'verbose'} );
my $flag;
unless( Storable::nstore( $to_write, $stored ) ) {
error( loc("could not store %1!", $stored) );
$flag++;
}
return $flag ? 0 : 1;
}
sub __memory_storable_file {
my $self = shift;
my $conf = $self->configure_object;
my $path = shift or return;
### check if we can retrieve a frozen data structure with storable ###
my $storable = $conf->get_conf('storable')
? can_load( modules => {'Storable' => '0.0'} )
: 0;
return unless $storable;
### $stored is the name of the frozen data structure ###
### changed to use File::Spec->catfile -jmb
my $stored = File::Spec->rel2abs(
File::Spec->catfile(
$path, #base dir
$conf->_get_source('stored') #file
. '.s' .
$Storable::VERSION #the version of storable
. '.c' .
$self->VERSION #the version of CPANPLUS
. STORABLE_EXT #append a suffix
)
);
return $stored;
}
# Local variables:
# c-indentation-style: bsd
# c-basic-offset: 4
# indent-tabs-mode: nil
# End:
# vim: expandtab shiftwidth=4:
1;

View File

@@ -0,0 +1,382 @@
package CPANPLUS::Internals::Source::SQLite;
use strict;
use warnings;
use base 'CPANPLUS::Internals::Source';
use CPANPLUS::Error;
use CPANPLUS::Internals::Constants;
use CPANPLUS::Internals::Source::SQLite::Tie;
use Data::Dumper;
use DBIx::Simple;
use DBD::SQLite;
use Params::Check qw[allow check];
use Locale::Maketext::Simple Class => 'CPANPLUS', Style => 'gettext';
use vars qw[$VERSION];
$VERSION = "0.9910";
use constant TXN_COMMIT => 1000;
=head1 NAME
CPANPLUS::Internals::Source::SQLite - SQLite implementation
=cut
{ my $Dbh;
my $DbFile;
sub __sqlite_file {
return $DbFile if $DbFile;
my $self = shift;
my $conf = $self->configure_object;
$DbFile = File::Spec->catdir(
$conf->get_conf('base'),
SOURCE_SQLITE_DB
);
return $DbFile;
};
sub __sqlite_dbh {
return $Dbh if $Dbh;
my $self = shift;
$Dbh = DBIx::Simple->connect(
"dbi:SQLite:dbname=" . $self->__sqlite_file,
'', '',
{ AutoCommit => 1 }
);
#$Dbh->dbh->trace(1);
$Dbh->query(qq{PRAGMA synchronous = OFF});
return $Dbh;
};
sub __sqlite_disconnect {
return unless $Dbh;
$Dbh->disconnect;
$Dbh = undef;
return;
}
}
{ my $used_old_copy = 0;
sub _init_trees {
my $self = shift;
my $conf = $self->configure_object;
my %hash = @_;
my($path,$uptodate,$verbose,$use_stored);
my $tmpl = {
path => { default => $conf->get_conf('base'), store => \$path },
verbose => { default => $conf->get_conf('verbose'), store => \$verbose },
uptodate => { required => 1, store => \$uptodate },
use_stored => { default => 1, store => \$use_stored },
};
check( $tmpl, \%hash ) or return;
### if it's not uptodate, or the file doesn't exist, we need to create
### a new sqlite db
if( not $uptodate or not -e $self->__sqlite_file ) {
$used_old_copy = 0;
### chuck the file
$self->__sqlite_disconnect;
1 while unlink $self->__sqlite_file;
### and create a new one
$self->__sqlite_create_db or do {
error(loc("Could not create new SQLite DB"));
return;
}
} else {
$used_old_copy = 1;
}
### set up the author tree
{ my %at;
tie %at, 'CPANPLUS::Internals::Source::SQLite::Tie',
dbh => $self->__sqlite_dbh, table => 'author',
key => 'cpanid', cb => $self;
$self->_atree( \%at );
}
### set up the author tree
{ my %mt;
tie %mt, 'CPANPLUS::Internals::Source::SQLite::Tie',
dbh => $self->__sqlite_dbh, table => 'module',
key => 'module', cb => $self;
$self->_mtree( \%mt );
}
### start a transaction
$self->__sqlite_dbh->query('BEGIN');
return 1;
}
sub _standard_trees_completed { return $used_old_copy }
sub _custom_trees_completed { return }
### finish transaction
sub _finalize_trees { $_[0]->__sqlite_dbh->commit; return 1 }
### saves current memory state, but not implemented in sqlite
sub _save_state {
error(loc("%1 has not implemented writing state to disk", __PACKAGE__));
return;
}
}
{ my $txn_count = 0;
### XXX move this outside the sub, so we only compute it once
my $class;
my @keys = qw[ author cpanid email ];
my $tmpl = {
class => { default => 'CPANPLUS::Module::Author', store => \$class },
map { $_ => { required => 1 } } @keys
};
### dbix::simple's expansion of (??) is REALLY expensive, so do it manually
my $ph = join ',', map { '?' } @keys;
sub _add_author_object {
my $self = shift;
my %hash = @_;
my $dbh = $self->__sqlite_dbh;
my $href = do {
local $Params::Check::NO_DUPLICATES = 1;
local $Params::Check::SANITY_CHECK_TEMPLATE = 0;
check( $tmpl, \%hash ) or return;
};
### keep counting how many we inserted
unless( ++$txn_count % TXN_COMMIT ) {
#warn "Committing transaction $txn_count";
$dbh->commit or error( $dbh->error ); # commit previous transaction
$dbh->begin_work or error( $dbh->error ); # and start a new one
}
$dbh->query(
"INSERT INTO author (". join(',',keys(%$href)) .") VALUES ($ph)",
values %$href
) or do {
error( $dbh->error );
return;
};
return 1;
}
}
{ my $txn_count = 0;
### XXX move this outside the sub, so we only compute it once
my $class;
my @keys = qw[ module version path comment author package description dslip mtime ];
my $tmpl = {
class => { default => 'CPANPLUS::Module', store => \$class },
map { $_ => { required => 1 } } @keys
};
### dbix::simple's expansion of (??) is REALLY expensive, so do it manually
my $ph = join ',', map { '?' } @keys;
sub _add_module_object {
my $self = shift;
my %hash = @_;
my $dbh = $self->__sqlite_dbh;
my $href = do {
local $Params::Check::NO_DUPLICATES = 1;
local $Params::Check::SANITY_CHECK_TEMPLATE = 0;
check( $tmpl, \%hash ) or return;
};
### fix up author to be 'plain' string
$href->{'author'} = $href->{'author'}->cpanid;
### keep counting how many we inserted
unless( ++$txn_count % TXN_COMMIT ) {
#warn "Committing transaction $txn_count";
$dbh->commit or error( $dbh->error ); # commit previous transaction
$dbh->begin_work or error( $dbh->error ); # and start a new one
}
$dbh->query(
"INSERT INTO module (". join(',',keys(%$href)) .") VALUES ($ph)",
values %$href
) or do {
error( $dbh->error );
return;
};
return 1;
}
}
{ my %map = (
_source_search_module_tree
=> [ module => module => 'CPANPLUS::Module' ],
_source_search_author_tree
=> [ author => cpanid => 'CPANPLUS::Module::Author' ],
);
while( my($sub, $aref) = each %map ) {
no strict 'refs';
my($table, $key, $class) = @$aref;
*$sub = sub {
my $self = shift;
my %hash = @_;
my($list,$type);
my $tmpl = {
allow => { required => 1, default => [ ], strict_type => 1,
store => \$list },
type => { required => 1, allow => [$class->accessors()],
store => \$type },
};
check( $tmpl, \%hash ) or return;
### we aliased 'module' to 'name', so change that here too
$type = 'module' if $type eq 'name';
my $meth = $table .'_tree';
{
my $throw = $self->$meth;
}
my $dbh = $self->__sqlite_dbh;
my $res = $dbh->query( "SELECT * from $table" );
my @rv = map { $self->$meth( $_->{$key} ) }
grep { allow( $_->{$type} => $list ) } $res->hashes;
return @rv;
}
}
}
sub __sqlite_create_db {
my $self = shift;
my $dbh = $self->__sqlite_dbh;
### we can ignore the result/error; not all sqlite implementations
### support this
$dbh->query( qq[
DROP TABLE IF EXISTS author;
\n]
) or do {
msg( $dbh->error );
};
$dbh->query( qq[
DROP TABLE IF EXISTS module;
\n]
) or do {
msg( $dbh->error );
};
$dbh->query( qq[
/* the author information */
CREATE TABLE author (
id INTEGER PRIMARY KEY AUTOINCREMENT,
author varchar(255),
email varchar(255),
cpanid varchar(255)
);
\n]
) or do {
error( $dbh->error );
return;
};
$dbh->query( qq[
/* the module information */
CREATE TABLE module (
id INTEGER PRIMARY KEY AUTOINCREMENT,
module varchar(255),
version varchar(255),
path varchar(255),
comment varchar(255),
author varchar(255),
package varchar(255),
description varchar(255),
dslip varchar(255),
mtime varchar(255)
);
\n]
) or do {
error( $dbh->error );
return;
};
$dbh->query( qq[
/* the module index */
CREATE INDEX IX_module_module ON module (
module
);
\n]
) or do {
error( $dbh->error );
return;
};
$dbh->query( qq[
/* the version index */
CREATE INDEX IX_module_version ON module (
version
);
\n]
) or do {
error( $dbh->error );
return;
};
$dbh->query( qq[
/* the module-version index */
CREATE INDEX IX_module_module_version ON module (
module, version
);
\n]
) or do {
error( $dbh->error );
return;
};
return 1;
}
1;

View File

@@ -0,0 +1,142 @@
package CPANPLUS::Internals::Source::SQLite::Tie;
use strict;
use warnings;
use CPANPLUS::Error;
use CPANPLUS::Module;
use CPANPLUS::Module::Fake;
use CPANPLUS::Module::Author::Fake;
use CPANPLUS::Internals::Constants;
use Params::Check qw[check];
use Module::Load::Conditional qw[can_load];
use Locale::Maketext::Simple Class => 'CPANPLUS', Style => 'gettext';
use vars qw[@ISA $VERSION];
$VERSION = "0.9910";
require Tie::Hash;
push @ISA, 'Tie::StdHash';
sub TIEHASH {
my $class = shift;
my %hash = @_;
my $tmpl = {
dbh => { required => 1 },
table => { required => 1 },
key => { required => 1 },
cb => { required => 1 },
offset => { default => 0 },
};
my $args = check( $tmpl, \%hash ) or return;
my $obj = bless { %$args, store => {} } , $class;
return $obj;
}
sub FETCH {
my $self = shift;
my $key = shift or return;
my $dbh = $self->{dbh};
my $cb = $self->{cb};
my $table = $self->{table};
### did we look this one up before?
if( my $obj = $self->{store}->{$key} ) {
return $obj;
}
my $res = $dbh->query(
"SELECT * from $table where $self->{key} = ?", $key
) or do {
error( $dbh->error );
return;
};
my $href = $res->hash;
### get rid of the primary key
delete $href->{'id'};
### no results?
return unless keys %$href;
### expand author if needed
### XXX no longer generic :(
if( $table eq 'module' ) {
$href->{author} = $cb->author_tree( $href->{author } ) or return;
}
my $class = {
module => 'CPANPLUS::Module',
author => 'CPANPLUS::Module::Author',
}->{ $table };
my $obj = $self->{store}->{$key} = $class->new( %$href, _id => $cb->_id );
return $obj;
}
sub STORE {
my $self = shift;
my $key = shift;
my $val = shift;
$self->{store}->{$key} = $val;
}
1;
sub FIRSTKEY {
my $self = shift;
my $dbh = $self->{'dbh'};
my $res = $dbh->query(
"select $self->{key} from $self->{table} order by $self->{key} limit 1"
);
$self->{offset} = 0;
my $key = $res->flat->[0];
return $key;
}
sub NEXTKEY {
my $self = shift;
my $dbh = $self->{'dbh'};
my $res = $dbh->query(
"select $self->{key} from $self->{table} ".
"order by $self->{key} limit 1 offset $self->{offset}"
);
$self->{offset} +=1;
my $key = $res->flat->[0];
my $val = $self->FETCH( $key );
### use each() semantics
return wantarray ? ( $key, $val ) : $key;
}
sub EXISTS { !!$_[0]->FETCH( $_[1] ) }
sub SCALAR {
my $self = shift;
my $dbh = $self->{'dbh'};
my $res = $dbh->query( "select count(*) from $self->{table}" );
return $res->flat;
}
### intentionally left blank
sub DELETE { }
sub CLEAR { }