Merge pull request #1299 from metacpan/haarg/es5-compat

various fixes for Elasticsearch 5 compatibility
metacpan · Oct 25, 2024 · a7c947c · a7c947c
2 parents 859022a + b621a68
commit a7c947c
Show file tree

Hide file tree

Showing 59 changed files with 583 additions and 417 deletions.
diff --git a/cpanfile b/cpanfile
@@ -124,6 +124,7 @@ requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile)
 requires 'Scalar::Util', '1.62'; # Moose
 requires 'Search::Elasticsearch' => '8.12';
 requires 'Search::Elasticsearch::Client::2_0' => '6.81';
+requires 'Search::Elasticsearch::Client::5_0' => '6.81';
 requires 'Term::Choose', '1.754'; # Git::Helpers
 requires 'Throwable::Error';
 requires 'Term::Size::Any'; # for Catalyst

diff --git a/cpanfile.snapshot b/cpanfile.snapshot
@@ -6480,6 +6480,38 @@ DISTRIBUTIONS
       namespace::clean 0
       strict 0
       warnings 0
+  Search-Elasticsearch-Client-5_0-6.81
+    pathname: E/EZ/EZIMUEL/Search-Elasticsearch-Client-5_0-6.81.tar.gz
+    provides:
+      Search::Elasticsearch::Client::5_0 6.81
+      Search::Elasticsearch::Client::5_0::Bulk 6.81
+      Search::Elasticsearch::Client::5_0::Direct 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Cat 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Cluster 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Indices 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Ingest 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Nodes 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Snapshot 6.81
+      Search::Elasticsearch::Client::5_0::Direct::Tasks 6.81
+      Search::Elasticsearch::Client::5_0::Role::API 6.81
+      Search::Elasticsearch::Client::5_0::Role::Bulk 6.81
+      Search::Elasticsearch::Client::5_0::Role::Scroll 6.81
+      Search::Elasticsearch::Client::5_0::Scroll 6.81
+      Search::Elasticsearch::Client::5_0::TestServer 6.81
+    requirements:
+      Devel::GlobalDestruction 0
+      ExtUtils::MakeMaker 0
+      Moo 0
+      Moo::Role 0
+      Search::Elasticsearch 6.00
+      Search::Elasticsearch::Role::API 0
+      Search::Elasticsearch::Role::Client::Direct 0
+      Search::Elasticsearch::Role::Is_Sync 0
+      Search::Elasticsearch::Util 0
+      Try::Tiny 0
+      namespace::clean 0
+      strict 0
+      warnings 0
   Sereal-Decoder-5.004
     pathname: Y/YV/YVES/Sereal-Decoder-5.004.tar.gz
     provides:

diff --git a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm
@@ -7,6 +7,7 @@ extends 'Catalyst::Plugin::Session::Store';
 use MooseX::Types::ElasticSearch qw( ES );
 
 use MetaCPAN::Server::Config ();
+use MetaCPAN::Util           qw( true false );
 
 has _session_es => (
     is      => 'ro',
@@ -55,7 +56,7 @@ sub store_session_data {
             type    => $self->_session_es_type,
             id      => $sid,
             body    => $session,
-            refresh => 1,
+            refresh => true,
         );
     }
 }
@@ -68,7 +69,7 @@ sub delete_session_data {
                 index   => $self->_session_es_index,
                 type    => $self->_session_es_type,
                 id      => $sid,
-                refresh => 1,
+                refresh => true,
             );
         };
     }

diff --git a/lib/ElasticSearchX/Model/Document/Set.pm b/lib/ElasticSearchX/Model/Document/Set.pm
@@ -0,0 +1,47 @@
+package ElasticSearchX::Model::Document::Set;
+use strict;
+use warnings;
+
+use MetaCPAN::Model::Hacks;
+
+no warnings 'redefine';
+
+our %query_override;
+my $_build_query = \&_build_query;
+*_build_query = sub {
+    my $query = $_build_query->(@_);
+    %$query = ( %$query, %query_override, );
+    return $query;
+};
+
+our %qs_override;
+my $_build_qs = \&_build_qs;
+*_build_qs = sub {
+    my $qs = $_build_qs->(@_);
+    %$qs = ( %$qs, %qs_override, );
+    return $qs;
+};
+
+my $delete = \&delete;
+*delete = sub {
+    local %qs_override    = ( search_type => 'query_then_fetch' );
+    local %query_override = ( sort        => '_doc' );
+    return $delete->(@_);
+};
+
+my $get = \&get;
+*get = sub {
+    my ( $self, $args, $qs ) = @_;
+    if ( $self->es->api_version eq '2_0' ) {
+        goto &$get;
+    }
+    my %qs = %{ $qs || {} };
+    if ( my $fields = $self->fields ) {
+        $qs{_source} = $fields;
+        local $self->{fields};
+        return $get->( $self, $args, \%qs );
+    }
+    goto &$get;
+};
+
+1;
diff --git a/lib/MetaCPAN/API/Model/Cover.pm b/lib/MetaCPAN/API/Model/Cover.pm
@@ -2,6 +2,8 @@ package MetaCPAN::API::Model::Cover;
 
 use MetaCPAN::Moose;
 
+use MetaCPAN::Util qw(hit_total);
+
 with 'MetaCPAN::API::Model::Role::ES';
 
 sub find_release_coverage {
@@ -17,7 +19,7 @@ sub find_release_coverage {
             size  => 999,
         }
     );
-    $res->{hits}{total} or return {};
+    hit_total($res) or return {};
 
     return +{
         %{ $res->{hits}{hits}[0]{_source} },

diff --git a/lib/MetaCPAN/Document/File.pm b/lib/MetaCPAN/Document/File.pm
@@ -414,9 +414,8 @@ sub _build_suggest {
     $weight = 0 if $weight < 0;
 
     return +{
-        input   => [$doc],
-        payload => { doc_name => $doc },
-        weight  => $weight,
+        input  => [$doc],
+        weight => $weight,
     };
 }
 

diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm
@@ -6,7 +6,7 @@ use List::Util                qw( max );
 use MetaCPAN::Query::Favorite ();
 use MetaCPAN::Query::File     ();
 use MetaCPAN::Query::Release  ();
-use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false );
+use MetaCPAN::Util            qw( true false );
 
 extends 'ElasticSearchX::Model::Document::Set';
 
@@ -75,7 +75,8 @@ my @ROGUE_DISTRIBUTIONS = qw(
 
 sub find {
     my ( $self, $module ) = @_;
-    my @candidates = $self->index->type('file')->query( {
+
+    my $query = {
         bool => {
             must => [
                 { term => { indexed    => true } },
@@ -119,22 +120,38 @@ sub find {
                 },
             ],
         },
-    } )->sort( [
-        '_score',
-        { 'version_numified' => { order => 'desc' } },
-        { 'date'             => { order => 'desc' } },
-        { 'mime'             => { order => 'asc' } },
-        { 'stat.mtime'       => { order => 'desc' } }
-    ] )->search_type('dfs_query_then_fetch')->size(100)->all;
+    };
+
+    my $res = $self->es->search(
+        index       => $self->index->name,
+        type        => 'file',
+        search_type => 'dfs_query_then_fetch',
+        body        => {
+            query => $query,
+            sort  => [
+                '_score',
+                { 'version_numified' => { order => 'desc' } },
+                { 'date'             => { order => 'desc' } },
+                { 'mime'             => { order => 'asc' } },
+                { 'stat.mtime'       => { order => 'desc' } }
+            ],
+            _source => [
+                qw( documentation module.indexed module.authoried module.name )
+            ],
+            size => 100,
+        },
+    );
+
+    my @candidates = @{ $res->{hits}{hits} };
 
     my ($file) = grep {
-        grep { $_->indexed && $_->authorized && $_->name eq $module }
-            @{ $_->module || [] }
-    } grep { !$_->documentation || $_->documentation eq $module }
+        grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module }
+            @{ $_->{module} || [] }
+    } grep { !$_->{documentation} || $_->{documentation} eq $module }
         @candidates;
 
     $file ||= shift @candidates;
-    return $file ? $self->get( $file->id ) : undef;
+    return $file ? $self->get( $file->{_id} ) : undef;
 }
 
 sub find_pod {
@@ -265,15 +282,13 @@ sub history {
 
 sub autocomplete {
     my ( $self, @terms ) = @_;
-    my $query = join( q{ }, @terms );
-    return $self unless $query;
 
-    my $data = $self->search_type('dfs_query_then_fetch')->query( {
+    my $query = {
         bool => {
             must => [
                 {
                     multi_match => {
-                        query    => $query,
+                        query    => join( q{ }, @terms ),
                         type     => 'most_fields',
                         fields   => [ 'documentation', 'documentation.*' ],
                         analyzer => 'camelcase',
@@ -288,15 +303,21 @@ sub autocomplete {
             must_not =>
                 [ { terms => { distribution => \@ROGUE_DISTRIBUTIONS } }, ],
         },
-    } )->sort( [ '_score', 'documentation' ] );
-
-    $data = $data->fields( [qw(documentation release author distribution)] )
-        unless $self->fields;
-
-    $data = $data->source(0)->raw->all;
+    };
+
+    my $data = $self->es->search(
+        search_type => 'dfs_query_then_fetch',
+        index       => $self->index->name,
+        type        => 'file',
+        body        => {
+            query   => $query,
+            sort    => [ '_score', 'documentation' ],
+            _source => [qw( documentation release author distribution )],
+        },
+    );
 
-    single_valued_arrayref_to_scalar( $_->{fields} )
-        for @{ $data->{hits}{hits} };
+    # this is backcompat. we don't use this end point.
+    $_->{fields} = delete $_->{_source} for @{ $data->{hits}{hits} };
 
     return $data;
 }
@@ -307,8 +328,7 @@ sub autocomplete_suggester {
 
     my $search_size = 100;
 
-    my $suggestions
-        = $self->search_type('dfs_query_then_fetch')->es->suggest( {
+    my $suggestions = $self->es->suggest( {
         index => $self->index->name,
         body  => {
             documentation => {
@@ -319,7 +339,7 @@ sub autocomplete_suggester {
                 }
             }
         },
-        } );
+    } );
 
     my %docs;
 
@@ -328,14 +348,6 @@ sub autocomplete_suggester {
             ( $docs{ $suggest->{text} }, $suggest->{score} );
     }
 
-    my @fields = qw(
-        author
-        date
-        deprecated
-        distribution
-        documentation
-        release
-    );
     my $data = $self->es->search( {
         index => $self->index->name,
         type  => 'file',
@@ -355,17 +367,22 @@ sub autocomplete_suggester {
                     ],
                 }
             },
+            _source => [ qw(
+                author
+                date
+                deprecated
+                distribution
+                documentation
+                release
+            ) ],
+            size => $search_size,
         },
-        fields => \@fields,
-        size   => $search_size,
     } );
 
     my %valid = map {
-        my $got = $_->{fields};
-        my %record;
-        @record{@fields} = map { $got->{$_}[0] } @fields;
-        $record{name} = delete $record{documentation};       # rename
-        ( $_->{fields}{documentation}[0] => \%record );
+        my %record = %{ $_->{_source} };
+        $record{name} = delete $record{documentation};    # rename
+        ( $record{name} => \%record );
     } @{ $data->{hits}{hits} };
 
     # remove any exact match, it will be added later
@@ -393,7 +410,7 @@ sub autocomplete_suggester {
 sub find_changes_files {
     my ( $self, $author, $release ) = @_;
     my $result = $self->files_by_category( $author, $release, ['changelog'],
-        { fields => true } );
+        { _source => true } );
     my ($file) = @{ $result->{categories}{changelog} || [] };
     return $file;
 }

diff --git a/lib/MetaCPAN/Document/Mirror.pm b/lib/MetaCPAN/Document/Mirror.pm
@@ -4,7 +4,7 @@ use strict;
 use warnings;
 
 use Moose;
-use ElasticSearchX::Model::Document::Types qw( Location );
+use MooseX::Types::ElasticSearch qw( Location );
 use ElasticSearchX::Model::Document;
 
 use MetaCPAN::Types::TypeTiny qw( Dict Str );

diff --git a/lib/MetaCPAN/Document/Release/Set.pm b/lib/MetaCPAN/Document/Release/Set.pm
@@ -2,8 +2,6 @@ package MetaCPAN::Document::Release::Set;
 
 use Moose;
 
-use MetaCPAN::Util qw( single_valued_arrayref_to_scalar );
-
 use MetaCPAN::Query::Release ();
 
 extends 'ElasticSearchX::Model::Document::Set';

diff --git a/lib/MetaCPAN/Model/Hacks.pm b/lib/MetaCPAN/Model/Hacks.pm
@@ -0,0 +1,23 @@
+package MetaCPAN::Model::Hacks;
+use strict;
+use warnings;
+
+sub import {
+    my ( $caller, $caller_file ) = caller;
+
+    my $file = $caller      =~ s{::}{/}gr . '.pm';
+    my $dir  = $caller_file =~ s{/\Q$file\E\z}{}r;
+    local @INC = grep $_ ne $dir, @INC;
+    my $inc;
+    {
+        local $INC{$file};
+        delete $INC{$file};
+        require $file;
+        $inc = $INC{$file};
+    }
+    delete $INC{$file};
+    $INC{$file} = $inc;
+    return;
+}
+
+1;
diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm
@@ -222,7 +222,8 @@ sub _build_document {
         || $document->{abstract} eq 'null' );
 
     $document
-        = $self->index->type('release')->put( $document, { refresh => 1 } );
+        = $self->index->type('release')
+        ->put( $document, { refresh => true } );
 
     # create distribution if doesn't exist
     my $dist_count = $self->es->count(