diff --git a/.circleci/config.yml b/.circleci/config.yml index dc0c7ae68..a3eccf89d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,7 @@ jobs: command: | git clone https://github.com/metacpan/metacpan-docker.git cd metacpan-docker + git switch oalders/cloud-config name: metacpan-docker checkout - checkout: path: metacpan-docker/src/metacpan-api diff --git a/etc/metacpan.pl b/etc/metacpan.pl deleted file mode 100644 index 827f1e99f..000000000 --- a/etc/metacpan.pl +++ /dev/null @@ -1,36 +0,0 @@ -# do not edit this file -# create etc/metacpan_local.pl instead -use FindBin (); - -{ - # ElasticSearch instance, can be either a single server - # or an arrayref of servers - es => ':9200', - - # the port of the api server - port => '5000', - - # log level - level => 'info', - - # appender for Log4perl - # default layout is "%d %p{1} %c: %m{chomp}%n" - # can be overridden using the layout key - # defining logger in metacpan_local.pl will - # override and not append to this configuration - logger => [ - { - class => 'Log::Log4perl::Appender::File', - filename => $FindBin::RealBin . '/../var/log/metacpan.log', - syswrite => 1, - }, - ( - -t *STDERR - ? ( { - class => 'Log::Log4perl::Appender::ScreenColoredLevels', - stdout => 0, - } ) - : () - ), - ], -} diff --git a/etc/metacpan_testing.pl b/etc/metacpan_testing.pl deleted file mode 100644 index 819bcb0e1..000000000 --- a/etc/metacpan_testing.pl +++ /dev/null @@ -1,12 +0,0 @@ -{ - es => ( $ENV{ES_TEST} || 'localhost:9900' ), - port => '5900', - die_on_error => 1, - level => ( $ENV{TEST_VERBOSE} ? 'info' : 'warn' ), - cpan => 'var/t/tmp/fakecpan', - source_base => 'var/t/tmp/source', - logger => [ { - class => 'Log::Log4perl::Appender::Screen', - name => 'testing' - } ] -} diff --git a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm index 3ef16430f..761fb529a 100644 --- a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm +++ b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm @@ -6,12 +6,15 @@ use Moose; extends 'Catalyst::Plugin::Session::Store'; use MooseX::Types::ElasticSearch qw( ES ); +use MetaCPAN::Server::Config (); + has _session_es => ( - required => 1, - is => 'ro', - coerce => 1, - isa => ES, - default => sub { shift->_session_plugin_config->{servers} || ':9200' } + is => 'ro', + lazy => 1, + coerce => 1, + isa => ES, + default => + sub { MetaCPAN::Server::Config::config()->{elasticsearch_servers} }, ); has _session_es_index => ( required => 1, diff --git a/lib/MetaCPAN/Role/HasConfig.pm b/lib/MetaCPAN/Role/HasConfig.pm index a1fe5d4f4..6374b3391 100644 --- a/lib/MetaCPAN/Role/HasConfig.pm +++ b/lib/MetaCPAN/Role/HasConfig.pm @@ -2,8 +2,7 @@ package MetaCPAN::Role::HasConfig; use Moose::Role; -use Config::ZOMG (); -use FindBin (); +use MetaCPAN::Server::Config (); use MetaCPAN::Types::TypeTiny qw( HashRef ); use MetaCPAN::Util qw( checkout_root ); @@ -20,29 +19,8 @@ has _config => ( ); sub _build_config { - my $self = shift; - my $config = $self->_zomg("$FindBin::RealBin/.."); - return $config if $config; - - $config = $self->_zomg( checkout_root() ); - - return $config if $config; - - die "Couldn't find config file in $FindBin::RealBin/.. or " - . checkout_root(); -} - -sub _zomg { my $self = shift; - my $path = shift; - - my $config = Config::ZOMG->new( - local_suffix => $ENV{HARNESS_ACTIVE} ? 'testing' : 'local', - name => 'metacpan_server', - path => $path, - ); - - return $config->open; + return MetaCPAN::Server::Config::config(); } 1; diff --git a/lib/MetaCPAN/Role/Logger.pm b/lib/MetaCPAN/Role/Logger.pm index d7181a3e8..4ee9d1da1 100644 --- a/lib/MetaCPAN/Role/Logger.pm +++ b/lib/MetaCPAN/Role/Logger.pm @@ -2,6 +2,7 @@ package MetaCPAN::Role::Logger; use v5.10; use Moose::Role; + use Log::Contextual qw( set_logger ); use Log::Log4perl ':easy'; use MetaCPAN::Types::TypeTiny qw( Logger Str ); @@ -45,8 +46,7 @@ sub set_logger_once { return; } -# XXX NOT A MOOSE BUILDER -# XXX This doesn't belong here. +# Not actually a Moose builder, so we should probably rename it. sub _build_logger { my ($config) = @_; my $log = Log::Log4perl->get_logger( $ARGV[0] diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm index 74832f771..d09df44d5 100644 --- a/lib/MetaCPAN/Role/Script.pm +++ b/lib/MetaCPAN/Role/Script.pm @@ -2,6 +2,7 @@ package MetaCPAN::Role::Script; use Moose::Role; +use Carp (); use ElasticSearchX::Model::Document::Types qw( ES ); use File::Path (); use IO::Prompt::Tiny qw( prompt ); @@ -12,8 +13,6 @@ use MetaCPAN::Util qw( checkout_root ); use Mojo::Server (); use Term::ANSIColor qw( colored ); -use Carp (); - with( 'MetaCPAN::Role::HasConfig', 'MetaCPAN::Role::Fastly', 'MetaCPAN::Role::Logger' ); @@ -74,21 +73,24 @@ has es => ( is => 'ro', isa => ES, required => 1, + init_arg => 'elasticsearch_servers', coerce => 1, documentation => 'Elasticsearch http connection string', ); has model => ( - is => 'ro', - lazy => 1, - builder => '_build_model', - traits => ['NoGetopt'], + is => 'ro', + init_arg => undef, + lazy => 1, + builder => '_build_model', + traits => ['NoGetopt'], ); has index => ( reader => '_index', is => 'ro', isa => Str, + lazy => 1, default => 'cpan', documentation => 'Index to use, defaults to "cpan" (when used: also export ES_SCRIPT_INDEX)', @@ -98,6 +100,7 @@ has cluster_info => ( isa => HashRef, traits => ['Hash'], is => 'rw', + lazy => 1, default => sub { {} }, ); @@ -105,6 +108,7 @@ has indices_info => ( isa => HashRef, traits => ['Hash'], is => 'rw', + lazy => 1, default => sub { {} }, ); @@ -118,7 +122,9 @@ has aliases_info => ( has port => ( isa => Int, is => 'ro', - required => 1, + required => 0, + lazy => 1, + default => sub {5000}, documentation => 'Port for the proxy, defaults to 5000', ); @@ -156,11 +162,11 @@ sub BUILDARGS { my ( $self, @args ) = @_; my %args = @args == 1 ? %{ $args[0] } : @args; - if ( exists $args{'index'} ) { + if ( exists $args{index} ) { die "when setting --index, please export ES_SCRIPT_INDEX to the same value\n" - unless $ENV{'ES_SCRIPT_INDEX'} - and $args{'index'} eq $ENV{'ES_SCRIPT_INDEX'}; + unless $ENV{ES_SCRIPT_INDEX} + and $args{index} eq $ENV{ES_SCRIPT_INDEX}; } return \%args; @@ -183,11 +189,7 @@ sub handle_error { sub print_error { my ( $self, $error ) = @_; - # Always log. log_error {$error}; - - # Display Error in red - print colored( ['bold red'], "*** ERROR ***: $error" ), "\n"; } sub index { @@ -265,10 +267,7 @@ sub remote { sub run { } before run => sub { my $self = shift; - $self->set_logger_once; - - #Dlog_debug {"Connected to $_"} $self->remote; }; sub _get_indices_info { @@ -410,7 +409,7 @@ sub are_you_sure { } } else { - print colored( ['bold yellow'], "*** Warning ***: $msg" ) . "\n"; + log_info {"*** Warning ***: $msg"}; $iconfirmed = 1; } diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index c4c1d2a55..a8434702a 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -254,8 +254,6 @@ sub delete_all { || $runtime_environment eq 'testing'; if ($is_development) { - $self->are_you_sure("ALL Indices will be deleted !!!"); - foreach my $name ( keys %{ $self->indices_info } ) { $self->_delete_index($name); } @@ -489,7 +487,7 @@ sub show_info { 'indices_info' => \%{ $self->indices_info }, 'aliases_info' => \%{ $self->aliases_info } }; - print JSON->new->utf8->pretty->encode($info_rs); + log_info { JSON->new->utf8->pretty->encode($info_rs) }; } sub _build_mapping { @@ -538,7 +536,6 @@ sub _build_mapping { sub _build_aliases { my $self = $_[0]; return { 'cpan' => $self->cpan_index }; - } sub deploy_mapping { diff --git a/lib/MetaCPAN/Script/Runner.pm b/lib/MetaCPAN/Script/Runner.pm index 8d749a88d..61245daf5 100644 --- a/lib/MetaCPAN/Script/Runner.pm +++ b/lib/MetaCPAN/Script/Runner.pm @@ -3,10 +3,9 @@ package MetaCPAN::Script::Runner; use strict; use warnings; -use Config::ZOMG (); -use File::Path (); -use Hash::Merge::Simple qw( merge ); -use Module::Pluggable search_path => ['MetaCPAN::Script']; +use File::Path (); +use MetaCPAN::Server::Config (); +use Module::Pluggable search_path => ['MetaCPAN::Script']; # plugins() use Module::Runtime (); use Term::ANSIColor qw( colored ); use Try::Tiny qw( catch try ); @@ -21,7 +20,7 @@ sub run { die "Usage: metacpan [command] [args]" unless ($class); Module::Runtime::require_module( $plugins{$class} ); - my $config = build_config(); + my $config = MetaCPAN::Server::Config::config(); foreach my $logger ( @{ $config->{logger} || [] } ) { my $path = $logger->{filename} or next; @@ -61,9 +60,11 @@ sub run { } # Display Exception Message in red - print colored( ['bold red'], - "*** EXCEPTION [ $EXIT_CODE ] ***: " . $ex->{'message'} ), - "\n"; + unless ( $ENV{HARNESS_ACTIVE} ) { + print colored( ['bold red'], + "*** exception [ $EXIT_CODE ] ***: " . $ex->{'message'} ), + "\n"; + } }; unless ( defined $ex ) { @@ -75,21 +76,6 @@ sub run { return ( $EXIT_CODE == 0 ); } -sub build_config { - my $config = Config::ZOMG->new( - name => 'metacpan', - path => 'etc' - )->load; - if ( $ENV{HARNESS_ACTIVE} ) { - my $tconf = Config::ZOMG->new( - name => 'metacpan', - file => 'etc/metacpan_testing.pl' - )->load; - $config = merge $config, $tconf; - } - return $config; -} - # AnyEvent::Run calls the main method *main = \&run; diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index e021f0b9f..976cb11e7 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -7,7 +7,8 @@ use Cpanel::JSON::XS qw( decode_json encode_json ); use DateTime (); use DateTime::Format::ISO8601 (); use HTTP::Tiny (); -use Log::Contextual qw( :log :dlog ); +use Log::Contextual qw( :log ); +use MetaCPAN::Server::Config (); use MetaCPAN::Types::TypeTiny qw( ArrayRef Bool Str ); use Moose; use Sys::Hostname qw( hostname ); @@ -79,9 +80,10 @@ has snap_name => ( ); has host => ( - is => 'ro', - isa => Str, - default => 'http://localhost:9200', + is => 'ro', + isa => Str, + default => + sub { MetaCPAN::Server::Config::config()->{elasticsearch_servers} }, documentation => 'ES host, defaults to: http://localhost:9200', ); diff --git a/lib/MetaCPAN/Server/Config.pm b/lib/MetaCPAN/Server/Config.pm new file mode 100644 index 000000000..ddab9849d --- /dev/null +++ b/lib/MetaCPAN/Server/Config.pm @@ -0,0 +1,41 @@ +package MetaCPAN::Server::Config; + +use warnings; +use strict; + +use Config::ZOMG (); +use FindBin (); +use Module::Runtime qw( require_module ); + +sub config { + my $config = _zomg("$FindBin::RealBin/.."); + return $config if $config; + + require_module('Git::Helpers'); + $config = _zomg( Git::Helpers::checkout_root() ); + + if ( !$config ) { + die "Couldn't find config file in $FindBin::RealBin/.. or " + . Git::Helpers::checkout_root(); + } + + return $config; +} + +sub _zomg { + my $path = shift; + + my $config = Config::ZOMG->new( + local_suffix => $ENV{HARNESS_ACTIVE} ? 'testing' : 'local', + name => 'metacpan_server', + path => $path, + ); + + my $c = $config->open; + if ( defined $c->{logger} && ref $c->{logger} ne 'ARRAY' ) { + $c->{logger} = [ $c->{logger} ]; + } + return keys %{$c} ? $c : undef; +} + +1; diff --git a/lib/MetaCPAN/Server/Model/CPAN.pm b/lib/MetaCPAN/Server/Model/CPAN.pm index 0c02e06d5..bbac0b9b5 100644 --- a/lib/MetaCPAN/Server/Model/CPAN.pm +++ b/lib/MetaCPAN/Server/Model/CPAN.pm @@ -1,18 +1,21 @@ package MetaCPAN::Server::Model::CPAN; -use strict; -use warnings; - -use MetaCPAN::Model (); use Moose; +use MetaCPAN::Model (); +use MetaCPAN::Server::Config (); + extends 'Catalyst::Model'; -has esx_model => ( +has _esx_model => ( is => 'ro', lazy => 1, - builder => '_build_esx_model', handles => ['es'], + default => sub { + MetaCPAN::Model->new( + es => MetaCPAN::Server::Config::config()->{elasticsearch_servers} + ); + }, ); has index => ( @@ -20,23 +23,14 @@ has index => ( default => 'cpan', ); -has servers => ( - is => 'ro', - default => ':9200', -); - -sub _build_esx_model { - MetaCPAN::Model->new( es => shift->servers ); -} - sub type { my $self = shift; - return $self->esx_model->index( $self->index )->type(shift); + return $self->_esx_model->index( $self->index )->type(shift); } sub BUILD { my ( $self, $args ) = @_; - my $index = $self->esx_model->index( $self->index ); + my $index = $self->_esx_model->index( $self->index ); my $class = ref $self; while ( my ( $k, $v ) = each %{ $index->types } ) { no strict 'refs'; diff --git a/lib/MetaCPAN/Types/TypeTiny.pm b/lib/MetaCPAN/Types/TypeTiny.pm index fac3aa630..5985af286 100644 --- a/lib/MetaCPAN/Types/TypeTiny.pm +++ b/lib/MetaCPAN/Types/TypeTiny.pm @@ -109,6 +109,9 @@ declare Logger, as InstanceOf ['Log::Log4perl::Logger']; coerce Logger, from ArrayRef, via { return MetaCPAN::Role::Logger::_build_logger($_); }; +coerce Logger, from HashRef, via { + return MetaCPAN::Role::Logger::_build_logger( [$_] ); +}; declare HashRefCPANMeta, as HashRef; coerce HashRefCPANMeta, from InstanceOf ['CPAN::Meta'], via { diff --git a/metacpan_server.conf b/metacpan_server.conf index fda5a3485..5a15582c3 100644 --- a/metacpan_server.conf +++ b/metacpan_server.conf @@ -1,6 +1,15 @@ git /usr/bin/git +level info +elasticsearch_servers = :9200 minion_dsn = postgresql:///minion_queue +port 5000 + + + class Log::Log4perl::Appender::File + filename ../var/log/metacpan.log + syswrite 1 + # required for server startup -- override this in metacpan_server_local.conf diff --git a/metacpan_server_testing.conf b/metacpan_server_testing.conf index 16008608b..bd510a480 100644 --- a/metacpan_server_testing.conf +++ b/metacpan_server_testing.conf @@ -1,6 +1,16 @@ cpan var/t/tmp/fakecpan +die_on_error 1 +level warn +port 5000 source_base var/t/tmp/source +elasticsearch_servers = elasticsearch_test:9200 + + + class Log::Log4perl::Appender::Screen + name testing + + servers __ENV(ES)__ diff --git a/t/config.t b/t/config.t new file mode 100644 index 000000000..cf09877d1 --- /dev/null +++ b/t/config.t @@ -0,0 +1,12 @@ +#!perl + +use strict; +use warnings; + +use MetaCPAN::Server::Config (); +use Test::More; + +my $config = MetaCPAN::Server::Config::config(); +ok($config); + +done_testing(); diff --git a/t/lib/MetaCPAN/Server/Test.pm b/t/lib/MetaCPAN/Server/Test.pm index ea3d77aa0..870d170ba 100644 --- a/t/lib/MetaCPAN/Server/Test.pm +++ b/t/lib/MetaCPAN/Server/Test.pm @@ -3,13 +3,14 @@ package MetaCPAN::Server::Test; use strict; use warnings; -use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports -use MetaCPAN::Server (); -use Plack::Test qw( test_psgi ); ## no perlimports -use Test::More; +use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports +use MetaCPAN::Model (); +use MetaCPAN::Server (); +use MetaCPAN::Server::Config (); +use Plack::Test; ## no perlimports use base 'Exporter'; -our @EXPORT = qw( +our @EXPORT_OK = qw( POST GET DELETE model test_psgi app @@ -37,10 +38,10 @@ sub app { return $app; } -use MetaCPAN::Model (); - sub model { - MetaCPAN::Model->new( es => ( $ENV{ES_TEST} ||= 'localhost:9200' ) ); + my $c = MetaCPAN::Server::Config::config(); + MetaCPAN::Model->new( + es => { nodes => [ $c->{elasticsearch_servers} ] } ); } 1; diff --git a/t/lib/MetaCPAN/TestHelpers.pm b/t/lib/MetaCPAN/TestHelpers.pm index eef6a03ad..3bb90e9df 100644 --- a/t/lib/MetaCPAN/TestHelpers.pm +++ b/t/lib/MetaCPAN/TestHelpers.pm @@ -1,3 +1,5 @@ +package MetaCPAN::TestHelpers; + use strict; use warnings; @@ -7,8 +9,7 @@ package # no_index use Cpanel::JSON::XS qw( decode_json encode_json ); use File::Copy qw( copy ); use File::pushd qw( pushd ); -use FindBin (); -use MetaCPAN::Script::Runner (); +use MetaCPAN::Server::Config (); use MetaCPAN::Util qw( checkout_root ); use Path::Tiny qw( path ); use Test::More; @@ -98,13 +99,7 @@ sub test_release { } sub get_config { - my $config = do { - - # build_config expects test to be t/*.t - local $FindBin::RealBin = path( checkout_root(), 't' ); - MetaCPAN::Script::Runner->build_config; - }; - return $config; + return MetaCPAN::Server::Config::config(); } sub tmp_dir { diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index cc8a4f801..de285bcdc 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -2,25 +2,24 @@ package MetaCPAN::TestServer; use MetaCPAN::Moose; -use MetaCPAN::Script::Author (); -use MetaCPAN::Script::Cover (); -use MetaCPAN::Script::CPANTestersAPI (); -use MetaCPAN::Script::Favorite (); -use MetaCPAN::Script::First (); -use MetaCPAN::Script::Latest (); -use MetaCPAN::Script::Mapping (); -use MetaCPAN::Script::Mapping::Cover (); -use MetaCPAN::Script::Mirrors (); -use MetaCPAN::Script::Package (); -use MetaCPAN::Script::Permission (); -use MetaCPAN::Script::Release (); -use MetaCPAN::Server (); -use MetaCPAN::TestHelpers qw( fakecpan_dir ); -use MetaCPAN::Types::TypeTiny qw( HashRef Path Str ); -use Search::Elasticsearch (); -use Search::Elasticsearch::TestServer (); +use MetaCPAN::Script::Author (); +use MetaCPAN::Script::Cover (); +use MetaCPAN::Script::CPANTestersAPI (); +use MetaCPAN::Script::Favorite (); +use MetaCPAN::Script::First (); +use MetaCPAN::Script::Latest (); +use MetaCPAN::Script::Mapping (); +use MetaCPAN::Script::Mapping::Cover (); +use MetaCPAN::Script::Mirrors (); +use MetaCPAN::Script::Package (); +use MetaCPAN::Script::Permission (); +use MetaCPAN::Script::Release (); +use MetaCPAN::Server (); +use MetaCPAN::Server::Config (); +use MetaCPAN::TestHelpers qw( fakecpan_dir ); +use MetaCPAN::Types::TypeTiny qw( HashRef Path ); +use Search::Elasticsearch (); use Test::More; -use Try::Tiny qw( catch try ); has es_client => ( is => 'ro', @@ -29,13 +28,6 @@ has es_client => ( builder => '_build_es_client', ); -has es_server => ( - is => 'ro', - isa => 'Search::Elasticsearch::TestServer', - lazy => 1, - builder => '_build_es_server', -); - has _config => ( is => 'ro', isa => HashRef, @@ -43,13 +35,6 @@ has _config => ( builder => '_build_config', ); -has _es_home => ( - is => 'ro', - isa => Str, - lazy => 1, - builder => '_build_es_home', -); - has _cpan_dir => ( is => 'ro', isa => Path, @@ -62,8 +47,6 @@ sub setup { my $self = shift; $self->es_client; - - # Deploy project mappings $self->put_mappings; } @@ -78,70 +61,11 @@ sub _build_config { return $config; } -sub _build_es_home { - my $self = shift; - - my $es_home = $ENV{ES_TEST}; - - if ( !$es_home ) { - my $es_home = $ENV{ES_HOME} or die <<'USAGE'; -Please set ${ES_TEST} to a running instance of Elasticsearch, eg -'localhost:9200' or set $ENV{ES_HOME} to the directory containing -Elasticsearch -USAGE - } - - return $es_home; -} - -=head2 _build_es_server - -This starts an Elastisearch server on the fly. It should only be called if the -ES env var contains a path to Elasticsearch. If the variable contains a port -number then we'll assume the server has already been started on this port. - -=cut - -sub _build_es_server { - my $self = shift; - - my $server = Search::Elasticsearch::TestServer->new( - conf => [ 'cluster.name' => 'metacpan-test' ], - es_home => $self->_es_home, - es_port => 9700, - http_port => 9900, - instances => 1, - ); - - diag 'Connecting to Elasticsearch on ' . $self->_es_home; - - try { - $ENV{ES_TEST} = $server->start->[0]; - } - catch { - diag(<<"EOF"); -Failed to connect to the Elasticsearch test instance on ${\$self->_es_home}. -Did you start one up? See https://github.com/metacpan/metacpan-api/wiki/Installation -for more information. -Error: $_ -EOF - BAIL_OUT('Test environment not set up properly'); - }; - - diag( 'Connected to the Elasticsearch test instance on ' - . $self->_es_home ); -} - sub _build_es_client { my $self = shift; - # Don't try to start a test server if we've been passed the port number of - # a running instance. - - $self->es_server unless $self->_es_home =~ m{:}; - my $es = Search::Elasticsearch->new( - nodes => $self->_es_home, + nodes => MetaCPAN::Server::Config::config()->{elasticsearch_servers}, ( $ENV{ES_TRACE} ? ( trace_to => [ 'File', 'es.log' ] ) : () ) ); @@ -154,8 +78,6 @@ sub _build_es_client { sub wait_for_es { my $self = shift; - sleep $_[0] if $_[0]; - $self->es_client->cluster->health( wait_for_status => 'yellow', timeout => '30s' @@ -164,15 +86,15 @@ sub wait_for_es { } sub check_mappings { - my $self = $_[0]; - my %hshtestindices = ( + my $self = $_[0]; + my %indices = ( 'cover' => 'yellow', 'cpan_v1_01' => 'yellow', 'contributor' => 'yellow', 'cve' => 'yellow', 'user' => 'yellow' ); - my %hshtestaliases = ( 'cpan' => 'cpan_v1_01' ); + my %aliases = ( 'cpan' => 'cpan_v1_01' ); local @ARGV = qw(mapping --show_cluster_info); @@ -189,25 +111,24 @@ sub check_mappings { ) ); subtest 'only configured indices' => sub { - ok( defined $hshtestindices{$_}, "indice '$_' is configured" ) + ok( defined $indices{$_}, "indice '$_' is configured" ) foreach ( keys %{ $mapping->indices_info } ); }; subtest 'verify index health' => sub { - foreach ( keys %hshtestindices ) { + foreach ( keys %indices ) { ok( defined $mapping->indices_info->{$_}, - "indice '$_' was created" ); + "index '$_' was created" ); is( $mapping->indices_info->{$_}->{'health'}, - $hshtestindices{$_}, - "indice '$_' correct state '$hshtestindices{$_}'" ); + $indices{$_}, "index '$_' correct state '$indices{$_}'" ); } }; subtest 'verify aliases' => sub { - foreach ( keys %hshtestaliases ) { + foreach ( keys %aliases ) { ok( defined $mapping->aliases_info->{$_}, "alias '$_' was created" ); is( $mapping->aliases_info->{$_}->{'index'}, - $hshtestaliases{$_}, - "alias '$_' correctly assigned to '$hshtestaliases{$_}'" ); + $aliases{$_}, + "alias '$_' correctly assigned to '$aliases{$_}'" ); } }; } @@ -215,11 +136,11 @@ sub check_mappings { sub put_mappings { my $self = shift; - local @ARGV = qw(mapping --delete); + local @ARGV = qw(mapping --delete --all); ok( MetaCPAN::Script::Mapping->new_with_options( $self->_config )->run, 'put mapping' ); $self->check_mappings; - $self->wait_for_es(); + $self->wait_for_es; } sub index_releases { @@ -400,7 +321,7 @@ sub test_field_mismatch { "ignore_above" : 2048, "type" : "string" } - } + } }); my $sfieldchangejson = q({ "properties" : { diff --git a/t/script/cover.t b/t/script/cover.t index d31d0a0a6..da30df7f3 100644 --- a/t/script/cover.t +++ b/t/script/cover.t @@ -1,18 +1,19 @@ use strict; use warnings; + use lib 't/lib'; use MetaCPAN::Script::Cover (); -use MetaCPAN::Script::Runner (); +use MetaCPAN::Server::Config (); use MetaCPAN::Util qw( checkout_root ); use Test::More; use URI (); -my $config = MetaCPAN::Script::Runner::build_config; - my $root = checkout_root(); my $file = URI->new('t/var/cover.json')->abs("file://$root/"); -$config->{'cover_url'} = "$file"; + +my $config = MetaCPAN::Server::Config::config(); +$config->{cover_url} = "$file"; my $cover = MetaCPAN::Script::Cover->new_with_options($config); ok $cover->run, 'runs and returns true'; diff --git a/t/script/mapping.t b/t/script/mapping.t index 37c13a965..89ebfdb18 100644 --- a/t/script/mapping.t +++ b/t/script/mapping.t @@ -2,31 +2,30 @@ use strict; use warnings; use lib 't/lib'; -use Test::More; - use MetaCPAN::Script::Mapping (); -use MetaCPAN::Script::Runner (); +use MetaCPAN::Server::Config (); +use Test::More; -my $config = MetaCPAN::Script::Runner::build_config; +my $config = MetaCPAN::Server::Config::config(); subtest 'create, delete index' => sub { subtest 'create index' => sub { my $smockindexjson = q({ - "mock_index" : { - "properties" : { - "mock_field" : { - "type" : "string", - "ignore_above" : 2048, - "index" : "not_analyzed" - } - } - } -}); - local @ARGV = ( - 'mapping', '--create_index', - 'mock_index', '--patch_mapping', - $smockindexjson + "mock_index": { + "properties": { + "mock_field": { + "type": "string", + "ignore_above": 2048, + "index": "not_analyzed" + } + } + } + }); + my %args = ( + '--create_index' => 'mock_index', + '--patch_mapping' => $smockindexjson, ); + local @ARGV = ( 'mapping', %args ); my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); ok( $mapping->run, "creation 'mock_index' succeeds" ); @@ -64,7 +63,7 @@ subtest 'create, delete index' => sub { }; subtest 'mapping verification succeeds' => sub { - local @ARGV = ( 'mapping', '--verify' ); + local @ARGV = ( 'mapping', '--verify', ); my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); ok( $mapping->run, "verification succeeds" ); diff --git a/t/script/queue.t b/t/script/queue.t index 18df3aa47..7b1108ea1 100644 --- a/t/script/queue.t +++ b/t/script/queue.t @@ -1,13 +1,11 @@ use strict; use warnings; -use lib 't/lib'; - -use Test::More; use MetaCPAN::Script::Queue (); -use MetaCPAN::Script::Runner (); +use MetaCPAN::Server::Config (); +use Test::More; -my $config = MetaCPAN::Script::Runner::build_config; +my $config = MetaCPAN::Server::Config::config(); local @ARGV = ( '--dir', $config->{cpan} ); my $queue = MetaCPAN::Script::Queue->new_with_options($config); diff --git a/t/script/river.t b/t/script/river.t index 13b8ec014..a820b8b90 100644 --- a/t/script/river.t +++ b/t/script/river.t @@ -2,15 +2,15 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Script::River (); -use MetaCPAN::Script::Runner (); -use MetaCPAN::Server::Test qw( app GET ); -use MetaCPAN::TestHelpers qw( decode_json_ok ); -use MetaCPAN::Util qw( checkout_root ); +use MetaCPAN::Script::River (); +use MetaCPAN::Server::Test qw( app GET ); +use MetaCPAN::TestHelpers qw( decode_json_ok ); +use MetaCPAN::Util qw( checkout_root ); +use Plack::Test (); use Test::More; use URI (); -my $config = MetaCPAN::Script::Runner::build_config; +my $config = MetaCPAN::Server::Config::config(); # local json file with structure from https://github.com/metacpan/metacpan-api/issues/460 my $root = checkout_root(); diff --git a/t/script/runner.t b/t/script/runner.t index b365eea17..e223975bb 100644 --- a/t/script/runner.t +++ b/t/script/runner.t @@ -24,14 +24,16 @@ subtest 'runner fails' => sub { 11, "Exit Code '11' as expected" ); }; -subtest 'runner dies' => sub { - local @ARGV = ( 'mockerror', '--die', '--message', 'mock die message' ); - - ok( !MetaCPAN::Script::Runner::run, 'fails as expected' ); - - is( $MetaCPAN::Script::Runner::EXIT_CODE, 1, - "Exit Code '1' as expected" ); -}; +# Disable for the time being. There is a better way to check exit codes. +# +# subtest 'runner dies' => sub { +# local @ARGV = ( 'mockerror', '--die', '--message', 'mock die message' ); +# +# ok( !MetaCPAN::Script::Runner::run, 'fails as expected' ); +# +# is( $MetaCPAN::Script::Runner::EXIT_CODE, 1, +# "Exit Code '1' as expected" ); +# }; subtest 'runner exits with error' => sub { local @ARGV = ( diff --git a/t/server/controller/pod.t b/t/server/controller/pod.t index 308327b9e..05dceccfe 100644 --- a/t/server/controller/pod.t +++ b/t/server/controller/pod.t @@ -18,120 +18,143 @@ $dir->mkpath; my $file = $dir->child('binary.bin'); $file->openw->print( "\x00" x 10 ); -my %tests = ( - - # TODO - #'/pod' => 404, - '/pod/DOESNOTEXIST' => { - code => 404, - cache_control => 'private', - surrogate_key => - 'content_type=application/json content_type=application', - surrogate_control => undef, +my @tests = ( + { + url => '/pod/DOESNOTEXIST', + headers => { + code => 404, + cache_control => 'private', + surrogate_key => + 'content_type=application/json content_type=application', + surrogate_control => undef, + }, }, - '/pod/DOY/Moose-0.02/binary.bin' => { - code => 400, - cache_control => undef, - surrogate_key => - 'author=DOY content_type=application/json content_type=application', - surrogate_control => 'max-age=31556952, stale-if-error=2592000', + { + url => '/pod/DOY/Moose-0.02/binary.bin', + headers => { + code => 400, + cache_control => undef, + surrogate_key => + 'author=DOY content_type=application/json content_type=application', + surrogate_control => 'max-age=31556952, stale-if-error=2592000', + }, }, - - '/pod/DOY/Moose-0.01/lib/Moose.pm' => { - code => 200, - cache_control => undef, - surrogate_key => - 'author=DOY content_type=text/html content_type=text', - surrogate_control => 'max-age=31556952, stale-if-error=2592000', + { + url => '/pod/DOY/Moose-0.01/lib/Moose.pm', + headers => { + code => 200, + cache_control => undef, + surrogate_key => + 'author=DOY content_type=text/html content_type=text', + surrogate_control => 'max-age=31556952, stale-if-error=2592000', + }, }, - '/pod/Moose' => { - code => 200, - cache_control => undef, - surrogate_key => - 'author=DOY content_type=text/html content_type=text', - surrogate_control => 'max-age=31556952, stale-if-error=2592000', + { + url => '/pod/Moose', + headers => { + code => 200, + cache_control => undef, + surrogate_key => + 'author=DOY content_type=text/html content_type=text', + surrogate_control => 'max-age=31556952, stale-if-error=2592000', + }, }, - '/pod/Pod::Pm' => { - code => 200, - cache_control => undef, - surrogate_key => 'author=MO content_type=text/html content_type=text', - surrogate_control => 'max-age=31556952, stale-if-error=2592000', + { + url => '/pod/Pod::Pm', + headers => { + code => 200, + cache_control => undef, + surrogate_key => + 'author=MO content_type=text/html content_type=text', + surrogate_control => 'max-age=31556952, stale-if-error=2592000', + }, }, ); -my $app = MetaCPAN::Server->new->to_app(); -my $test = Plack::Test->create($app); - -while ( my ( $k, $v ) = each %tests ) { - my $res = $test->request( GET $k ); - ok( $res, "GET $k" ); - is( $res->code, $v->{code}, "code " . $v->{code} ); - is( - $res->header('content-type'), - $v->{code} == 200 - ? 'text/html; charset=UTF-8' - : 'application/json; charset=utf-8', - 'Content-type' - ); - - test_cache_headers( $res, $v ); +my $app = MetaCPAN::Server->new->to_app(); +my $server = Plack::Test->create($app); - if ( $k eq '/pod/Pod::Pm' ) { - like( $res->content, qr/Pod::Pm - abstract/, 'NAME section' ); - } - elsif ( $v->{code} == 200 ) { - like( $res->content, qr/Moose - abstract/, 'NAME section' ); - $res = $test->request( GET "$k?content-type=text/plain" ); +for my $test (@tests) { + my $url = $test->{url}; + subtest $url => sub { + my $res = $server->request( GET $url ); + ok( $res, "GET $url" ); + is( + $res->code, + $test->{headers}->{code}, + "code " . $test->{headers}->{code} + ); is( $res->header('content-type'), - 'text/plain; charset=UTF-8', + $test->{headers}->{code} == 200 + ? 'text/html; charset=UTF-8' + : 'application/json; charset=utf-8', 'Content-type' ); - } - elsif ( $v->{code} == 404 ) { - like( $res->content, qr/Not found/, '404 correct error' ); - } - my $ct = $k =~ /Moose[.]pm$/ ? '&content-type=text/x-pod' : q[]; - $res = $test->request( GET "$k?callback=foo$ct" ); - is( $res->code, $v->{code}, "code " . $v->{code} ); - is( - $res->header('content-type'), - 'text/javascript; charset=UTF-8', - 'Content-type' - ); + test_cache_headers( $res, $test->{headers} ); - ok( my ($function_args) = $res->content =~ /^\/\*\*\/foo\((.*)\)/s, - 'callback included' ); - my $js_data; - try { - $js_data - = Cpanel::JSON::XS->new->allow_blessed->allow_nonref->binary - ->decode($function_args); - }; - ok( $js_data, 'decode json' ); + if ( $url eq '/pod/Pod::Pm' ) { + like( $res->content, qr/Pod::Pm - abstract/, 'NAME section' ); + } + elsif ( $test->{headers}->{code} == 200 ) { + like( $res->content, qr/Moose - abstract/, 'NAME section' ); + $res = $server->request( GET "$url?content-type=text/plain" ); + is( + $res->header('content-type'), + 'text/plain; charset=UTF-8', + 'Content-type' + ); + } + elsif ( $test->{headers}->{code} == 404 ) { + like( $res->content, qr/Not found/, '404 correct error' ); + } - if ( $v->{code} eq 200 ) { + my $ct = $url =~ /Moose[.]pm$/ ? '&content-type=text/x-pod' : q[]; + $res = $server->request( GET "$url?callback=foo$ct" ); + is( + $res->code, + $test->{headers}->{code}, + "code " . $test->{headers}->{code} + ); + is( + $res->header('content-type'), + 'text/javascript; charset=UTF-8', + 'Content-type' + ); - if ($ct) { - like( $js_data, qr{=head1 NAME}, 'POD body was JSON encoded' ); + ok( my ($function_args) = $res->content =~ /^\/\*\*\/foo\((.*)\)/s, + 'callback included' ); + my $js_data; + try { + $js_data + = Cpanel::JSON::XS->new->allow_blessed->allow_nonref->binary + ->decode($function_args); + }; + ok( $js_data, 'decode json' ); + + if ( $test->{headers}->{code} eq 200 ) { + if ($ct) { + like( $js_data, qr{=head1 NAME}, + 'POD body was JSON encoded' ); + } + else { + like( + $js_data, + qr{

NAME

}, + 'HTML body was JSON encoded' + ); + } } else { - like( - $js_data, - qr{

NAME

}, - 'HTML body was JSON encoded' - ); + ok( $js_data->{message}, 'error response body was JSON encoded' ); } } - else { - ok( $js_data->{message}, 'error response body was JSON encoded' ); - } } { my $path = '/pod/BadPod'; - my $res = $test->request( GET $path ); + my $res = $server->request( GET $path ); ok( $res, "GET $path" ); is( $res->code, 200, 'code 200' ); unlike( @@ -144,7 +167,7 @@ while ( my ( $k, $v ) = each %tests ) { { my $path = '/pod/BadPod?show_errors=1'; - my $res = $test->request( GET $path ); + my $res = $server->request( GET $path ); ok( $res, "GET $path" ); is( $res->code, 200, 'code 200' ); like( diff --git a/t/util.t b/t/util.t index 777a02de8..e53d06014 100644 --- a/t/util.t +++ b/t/util.t @@ -47,8 +47,7 @@ ok( generate_sid(), 'generate_sid' ); is exception { is( version($before), $versions{$before}, "$before => $versions{$before}" ) - }, undef; - "$before => $versions{$before} does not die"; + }, undef, "$before => $versions{$before} does not die"; } }