From c975bf52b4bab39b0cf050a1654f777f616f3046 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Tue, 17 Dec 2024 22:35:14 +0100 Subject: [PATCH] move find and find_pod methods to query --- lib/MetaCPAN/Document/File/Set.pm | 100 ------------------- lib/MetaCPAN/Query/File.pm | 118 +++++++++++++++++++++++ lib/MetaCPAN/Server/Controller/Module.pm | 8 +- lib/MetaCPAN/Server/Controller/Pod.pm | 5 +- lib/MetaCPAN/Server/Controller/Source.pm | 7 +- lib/MetaCPAN/Server/Role/Request.pm | 6 ++ t/lib/MetaCPAN/Server/Test.pm | 28 ++++-- t/release/moose.t | 6 -- t/release/pm-PL.t | 30 +++--- t/release/pod-pm.t | 10 +- t/release/versions.t | 11 ++- 11 files changed, 185 insertions(+), 144 deletions(-) diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index b289d84eb..3e3cd66bb 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -7,106 +7,6 @@ use MetaCPAN::Util qw( true false ); extends 'ElasticSearchX::Model::Document::Set'; -sub find { - my ( $self, $module ) = @_; - - my $query = { - bool => { - must => [ - { term => { indexed => true } }, - { term => { authorized => true } }, - { term => { status => 'latest' } }, - { - bool => { - should => [ - { term => { documentation => $module } }, - { - nested => { - path => "module", - query => { - bool => { - must => [ - { - term => { "module.name" => - $module } - }, - { - bool => { should => - [ - { term => - { "module.authorized" - => true - } }, - { exists => - { field => - 'module.associated_pod' - } }, - ], - } - }, - ], - }, - }, - } - }, - ] - } - }, - ], - }, - }; - - my $res = $self->es->search( - es_doc_path('file'), - search_type => 'dfs_query_then_fetch', - body => { - query => $query, - sort => [ - '_score', - { 'version_numified' => { order => 'desc' } }, - { 'date' => { order => 'desc' } }, - { 'mime' => { order => 'asc' } }, - { 'stat.mtime' => { order => 'desc' } } - ], - _source => [ - qw( documentation module.indexed module.authoried module.name ) - ], - size => 100, - }, - ); - - my @candidates = @{ $res->{hits}{hits} }; - - my ($file) = grep { - grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module } - @{ $_->{module} || [] } - } grep { !$_->{documentation} || $_->{documentation} eq $module } - @candidates; - - $file ||= shift @candidates; - return $file ? $self->get( $file->{_id} ) : undef; -} - -sub find_pod { - my ( $self, $name ) = @_; - my $file = $self->find($name); - return $file unless ($file); - my ($module) - = grep { $_->indexed && $_->authorized && $_->name eq $name } - @{ $file->module || [] }; - if ( $module && ( my $pod = $module->associated_pod ) ) { - my ( $author, $release, @path ) = split( /\//, $pod ); - return $self->get( { - author => $author, - release => $release, - path => join( '/', @path ), - } ); - } - else { - return $file; - } -} - =head2 history Find the history of a given module/documentation. diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index b6ad3aa6f..8e7a421ac 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -559,5 +559,123 @@ sub documented_modules { }; } +sub find_module { + my ( $self, $module, $fields ) = @_; + + my $query = { + bool => { + must => [ + { term => { indexed => true } }, + { term => { authorized => true } }, + { term => { status => 'latest' } }, + { + bool => { + should => [ + { term => { documentation => $module } }, + { + nested => { + path => "module", + query => { + bool => { + must => [ + { + term => { "module.name" => + $module } + }, + { + bool => { should => + [ + { term => + { "module.authorized" + => true + } }, + { exists => + { field => + 'module.associated_pod' + } }, + ], + } + }, + ], + }, + }, + } + }, + ] + } + }, + ], + }, + }; + + my $res = $self->es->search( + es_doc_path('file'), + search_type => 'dfs_query_then_fetch', + body => { + query => $query, + sort => [ + '_score', + { 'version_numified' => { order => 'desc' } }, + { 'date' => { order => 'desc' } }, + { 'mime' => { order => 'asc' } }, + { 'stat.mtime' => { order => 'desc' } } + ], + _source => [ + qw( documentation module.indexed module.authoried module.name ) + ], + size => 100, + }, + ); + + my @candidates = @{ $res->{hits}{hits} }; + + my ($file) = grep { + grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module } + @{ $_->{module} || [] } + } grep { !$_->{documentation} || $_->{documentation} eq $module } + @candidates; + + $file ||= shift @candidates; + return undef + if !$file; + return $self->es->get_source( + es_doc_path('file'), + id => $file->{_id}, + ( $fields ? ( _source => $fields ) : () ), + ); +} + +sub find_pod { + my ( $self, $name ) = @_; + my $file = $self->find_module($name); + return $file + unless $file; + my ($module) + = grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $name } + @{ $file->{module} || [] }; + if ( $module && ( my $pod = $module->{associated_pod} ) ) { + my ( $author, $release, @path ) = split( /\//, $pod ); + my $query = { + bool => { + must => [ + { term => { author => $author } }, + { term => { release => $release } }, + { term => { path => join( '/', @path ) } }, + ], + }, + }; + my $pod_file = $self->es->search( + es_doc_path('file'), + body => { + query => $query, + }, + ); + return $pod_file->{hits}{hits}[0]; + } + else { + return $file; + } +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Server/Controller/Module.pm b/lib/MetaCPAN/Server/Controller/Module.pm index b2a04cf54..db1b60dfd 100644 --- a/lib/MetaCPAN/Server/Controller/Module.pm +++ b/lib/MetaCPAN/Server/Controller/Module.pm @@ -12,14 +12,12 @@ has '+type' => ( default => 'file' ); sub get : Path('') : Args(1) { my ( $self, $c, $name ) = @_; - my $file = $self->model($c)->raw->find($name); + my $file + = $c->model('ESQuery')->file->find_module( $name, $c->req->fields ); if ( !defined $file ) { $c->detach( '/not_found', [] ); } - $c->stash( $file->{_source} - || single_valued_arrayref_to_scalar( $file->{fields} ) ) - || $c->detach( '/not_found', - ['The requested field(s) could not be found'] ); + $c->stash($file); } __PACKAGE__->meta->make_immutable(); diff --git a/lib/MetaCPAN/Server/Controller/Pod.pm b/lib/MetaCPAN/Server/Controller/Pod.pm index b001f6a4c..9f2edc84b 100644 --- a/lib/MetaCPAN/Server/Controller/Pod.pm +++ b/lib/MetaCPAN/Server/Controller/Pod.pm @@ -36,9 +36,10 @@ sub find : Path('') { sub get : Path('') : Args(1) { my ( $self, $c, $module ) = @_; - $module = $c->model('ESModel')->doc('file')->find_pod($module) + $module = $c->model('ESQuery')->file->find_pod($module) or $c->detach( '/not_found', [] ); - $c->forward( 'find', [ map { $module->$_ } qw(author release path) ] ); + $c->forward( 'find', + [ map { $module->{_source}{$_} } qw(author release path) ] ); } sub find_dist_links { diff --git a/lib/MetaCPAN/Server/Controller/Source.pm b/lib/MetaCPAN/Server/Controller/Source.pm index d667b3d5d..868e177cd 100644 --- a/lib/MetaCPAN/Server/Controller/Source.pm +++ b/lib/MetaCPAN/Server/Controller/Source.pm @@ -63,9 +63,12 @@ sub module : Chained('index') : PathPart('') : Args(1) { $c->cdn_never_cache(1); - $module = $c->model('ESModel')->doc('file')->find($module) + my $file + = $c->model('ESQuery') + ->file->find_module( $module, [qw(author release path)] ) or $c->detach( '/not_found', [] ); - $c->forward( 'get', [ map { $module->$_ } qw(author release path) ] ); + + $c->forward( 'get', [ map { $file->{$_} } qw(author release path) ] ); } 1; diff --git a/lib/MetaCPAN/Server/Role/Request.pm b/lib/MetaCPAN/Server/Role/Request.pm index 39623e135..943291265 100644 --- a/lib/MetaCPAN/Server/Role/Request.pm +++ b/lib/MetaCPAN/Server/Role/Request.pm @@ -14,4 +14,10 @@ around [qw(content_type header)] => sub { : $header; }; +sub fields { + my $self = shift; + my @fields = map { split /,/ } $self->param('fields'); + return @fields ? \@fields : undef; +} + 1; diff --git a/t/lib/MetaCPAN/Server/Test.pm b/t/lib/MetaCPAN/Server/Test.pm index 98ef26885..601e0ce5a 100644 --- a/t/lib/MetaCPAN/Server/Test.pm +++ b/t/lib/MetaCPAN/Server/Test.pm @@ -2,18 +2,22 @@ package MetaCPAN::Server::Test; use strict; use warnings; +use feature qw(state); -use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports -use MetaCPAN::Model (); -use MetaCPAN::Server (); -use MetaCPAN::Server::Config (); -use Plack::Test; ## no perlimports +use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports +use MetaCPAN::Model (); +use MetaCPAN::Server (); +use MetaCPAN::Server::Config (); +use MooseX::Types::ElasticSearch qw( ES ); +use Plack::Test; ## no perlimports use base 'Exporter'; our @EXPORT_OK = qw( POST GET DELETE + es model test_psgi app + query ); # Begin the load-order dance. @@ -38,9 +42,19 @@ sub app { return $app; } +sub es { + state $es = do { + my $c = MetaCPAN::Server::Config::config(); + ES->assert_coerce( $c->{elasticsearch_servers} ); + }; +} + sub model { - my $c = MetaCPAN::Server::Config::config(); - MetaCPAN::Model->new( es => $c->{elasticsearch_servers} ); + state $model = MetaCPAN::Model->new( es => es() ); +} + +sub query { + state $query = MetaCPAN::Query->new( es => es() ); } 1; diff --git a/t/release/moose.t b/t/release/moose.t index f3b28a03d..5582bfb8f 100644 --- a/t/release/moose.t +++ b/t/release/moose.t @@ -51,12 +51,6 @@ ok( is( $ppport->name, 'ppphdoc', 'name doesn\'t contain a dot' ); -ok( my $moose = $model->doc('file')->find('Moose'), 'find Moose module' ); - -is( $moose->name, 'Moose.pm', 'defined in Moose.pm' ); - -is( $moose->module->[0]->associated_pod, 'DOY/Moose-0.02/lib/Moose.pm' ); - my $signature; $signature = $model->doc('file')->query( { bool => { diff --git a/t/release/pm-PL.t b/t/release/pm-PL.t index 9abd3cd4a..3facfb9c5 100644 --- a/t/release/pm-PL.t +++ b/t/release/pm-PL.t @@ -2,40 +2,46 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( app GET model test_psgi ); +use MetaCPAN::ESConfig qw( es_doc_path ); +use MetaCPAN::Server::Test qw( app GET query es test_psgi ); use Test::More; -my $model = model(); +my $query = query(); # Module::Faker will generate a regular pm for the main module. -is( $model->doc('file')->find('uncommon::sense')->path, +is( $query->file->find_module('uncommon::sense')->{path}, 'lib/uncommon/sense.pm', 'find main module' ); # This should be the .pm.PL file we specified. -ok( my $pm = $model->doc('file')->find('less::sense'), +ok( my $pm = $query->file->find_module('less::sense'), 'find sense.pm.PL module' ); -is( $pm->name, 'sense.pm.PL', 'name is correct' ); +is( $pm->{name}, 'sense.pm.PL', 'name is correct' ); is( - $pm->module->[0]->associated_pod, + $pm->{module}->[0]->{associated_pod}, 'MO/uncommon-sense-0.01/sense.pod', 'has associated pod file' ); # Ensure that $VERSION really came from file and not dist. -is( $pm->module->[0]->version, +is( $pm->{module}->[0]->{version}, '4.56', 'pm.PL module version is (correctly) different than main dist' ) # TRAVIS 5.16 - or diag( Test::More::explain( $pm->meta->get_data($pm) ) ); + or diag($pm); { # Verify all the files we expect to be contained in the release. - my $files - = $model->doc('file') - ->query( { term => { release => 'uncommon-sense-0.01' } } ) - ->raw->size(20)->all->{hits}->{hits}; + my $files = es->search( + es_doc_path('file'), + body => { + query => { + term => { release => 'uncommon-sense-0.01' }, + }, + size => 20, + }, + )->{hits}->{hits}; $files = [ map { $_->{_source} } @$files ]; is_deeply( diff --git a/t/release/pod-pm.t b/t/release/pod-pm.t index b24a217cd..0730bafe3 100644 --- a/t/release/pod-pm.t +++ b/t/release/pod-pm.t @@ -2,18 +2,18 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( model ); +use MetaCPAN::Server::Test qw( query ); use Test::More; -my $model = model(); +my $query = query(); -ok( my $pod_pm = $model->doc('file')->find('Pod::Pm'), +ok( my $pod_pm = $query->file->find_module('Pod::Pm'), 'find Pod::Pm module' ); -is( $pod_pm->name, 'Pm.pm', 'defined in Pm.pm' ); +is( $pod_pm->{name}, 'Pm.pm', 'defined in Pm.pm' ); is( - $pod_pm->module->[0]->associated_pod, + $pod_pm->{module}->[0]->{associated_pod}, 'MO/Pod-Pm-0.01/lib/Pod/Pm.pod', 'has associated pod file' ); diff --git a/t/release/versions.t b/t/release/versions.t index 90f25ba9a..a08bb379b 100644 --- a/t/release/versions.t +++ b/t/release/versions.t @@ -2,10 +2,10 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( model ); +use MetaCPAN::Server::Test qw( query ); use Test::More; -my $model = model(); +my $query = query(); my %modules = ( 'Versions::Our' => '1.45', @@ -16,14 +16,15 @@ my %modules = ( while ( my ( $module, $version ) = each %modules ) { - ok( my $file = $model->doc('file')->find($module), "find $module" ) + ok( my $file = $query->file->find_module($module), "find $module" ) or next; ( my $path = "lib/$module.pm" ) =~ s/::/\//; - is( $file->path, $path, 'expected path' ); + is( $file->{path}, $path, 'expected path' ); # Check module version (different than dist version). - is( $file->module->[0]->version, $version, 'version parsed from file' ); + is( $file->{module}->[0]->{version}, + $version, 'version parsed from file' ); }