From 9d006cb5ddaf96aeff9fc9044db50071cb8a81e8 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Mon, 29 May 2023 10:25:11 +0530 Subject: [PATCH 1/8] ignoring column defaults of db which are function calls issue #124 currently read_schema returns the default values of db in verbatim even if they are function calls. This causes attempts to insert the function call body as defaults in columns instead of the the value, which does not makes sense. The current fix returns undef for such column defaults. Additionally as a measure of caution it does not do the above in case the default has been handled in fix_default mapping. --- lib/Yancy/Backend/Dbic.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/Yancy/Backend/Dbic.pm b/lib/Yancy/Backend/Dbic.pm index 998c6323..debe3533 100644 --- a/lib/Yancy/Backend/Dbic.pm +++ b/lib/Yancy/Backend/Dbic.pm @@ -305,6 +305,11 @@ sub read_schema { my $default = ref $c->{default_value} eq 'SCALAR' ? ${ $c->{default_value} } : $c->{default_value }; + + if ( $default && $default =~ m#[\w]+\s*\(.*\)# && ! exists $fix_default{$default} ) { + $default = undef; + } + $schema{ $schema_name }{ properties }{ $column } = { $self->_map_type( $c ), $is_auto ? ( readOnly => true ) : (), From a555e2c22856efe4b26fb83f9f38f0713a354ef3 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Mon, 29 May 2023 22:01:26 +0530 Subject: [PATCH 2/8] support for uuid data type column in search currently uuid columns are typed as string with no particular format. However uuid strings stored in uuid data types in databases does not support the like operator. It is better to use '=' operator for the uuid columns for a functional match. --- lib/Yancy/Backend/Dbic.pm | 3 +++ lib/Yancy/Controller/Yancy.pm | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/Yancy/Backend/Dbic.pm b/lib/Yancy/Backend/Dbic.pm index debe3533..43eb4440 100644 --- a/lib/Yancy/Backend/Dbic.pm +++ b/lib/Yancy/Backend/Dbic.pm @@ -399,6 +399,9 @@ sub _map_type { elsif ( $db_type =~ /(?:blob|bytea)/i ) { %conf = ( %conf, type => 'string', format => 'binary' ); } + elsif ( $db_type =~ /(?:uuid)/i ) { + %conf = ( %conf, type => 'string', format => 'uuid' ); + } else { # Default to string %conf = ( %conf, type => 'string' ); diff --git a/lib/Yancy/Controller/Yancy.pm b/lib/Yancy/Controller/Yancy.pm index 5596c346..070c2088 100644 --- a/lib/Yancy/Controller/Yancy.pm +++ b/lib/Yancy/Controller/Yancy.pm @@ -1272,8 +1272,12 @@ sub _get_list_args { for my $key ( @{ $c->req->params->names } ) { next unless exists $props->{ $key }; my $type = $props->{$key}{type} || 'string'; + my $format = $props->{$key}{format} // '' ; my $value = $c->param( $key ); - if ( is_type( $type, 'string' ) ) { + if ( is_type( $type, 'string' ) && $format eq 'uuid' ) { + $param_filter{ $key } = $value ; + } + elsif ( is_type( $type, 'string' ) ) { if ( ( $value =~ tr/*/%/ ) <= 0 ) { $value = "\%$value\%"; } From 24777062508a0599b42460016b5f99fbbfa52b7f Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Tue, 30 May 2023 23:05:22 +0530 Subject: [PATCH 3/8] intial support for joining relations and filtering on joined relations. support for joins have been implemented in list method. This will allow fetching of related records by specifying in 'join' parameters. currently joining with immediate relations is only supported. --- lib/Yancy/Controller/Yancy.pm | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/Yancy/Controller/Yancy.pm b/lib/Yancy/Controller/Yancy.pm index 070c2088..5457cce7 100644 --- a/lib/Yancy/Controller/Yancy.pm +++ b/lib/Yancy/Controller/Yancy.pm @@ -2,6 +2,7 @@ package Yancy::Controller::Yancy; our $VERSION = '1.089'; # ABSTRACT: Basic controller for displaying content +use Lingua::EN::Inflect::Phrase; =head1 SYNOPSIS use Mojolicious::Lite; @@ -1262,18 +1263,45 @@ sub _get_list_args { $opt->{order_by} = $order_by; } + my $joins = $c->every_param('join'); + if ( $joins && @$joins ) { + $opt->{ join } = $joins; + } + if ( my $join = $c->stash( 'join' ) ) { $opt->{ join } = $join; } my $schema = $c->schema; + my $model = $schema->model; + + my $related_props = {} ; + # enumerate all the properties of all the joins and store in related_props + # keyed by 'join_name.property_name' + foreach my $join ( @$joins) { + # derieve the schema_name from join name. + my $schema_name = Mojo::Util::camelize ( Lingua::EN::Inflect::Phrase::to_S ( $join ) ); + eval { + my $join_schema = $model -> schema($schema_name); # may raise exception if $schema_name is not derieved properly. + my $schema_properties = $join_schema->{json_schema}->{properties} ; + # popuplate related_props with 'join_name.property_name' + map { $related_props->{"$join.$_"} = $schema_properties->{$_} } keys %{$schema_properties} ; + } + } + my $props = $schema->json_schema->{properties}; my %param_filter = (); for my $key ( @{ $c->req->params->names } ) { - next unless exists $props->{ $key }; - my $type = $props->{$key}{type} || 'string'; - my $format = $props->{$key}{format} // '' ; + next unless ( exists $props->{ $key } || exists $related_props->{ $key } ) ; + my $type = $props->{$key}{type} || $related_props->{$key}{type} || 'string'; + my $format = $props->{$key}{format} || $related_props->{$key}{format} // '' ; my $value = $c->param( $key ); + + # if the key belongs to main relation prefix with 'me.' + if ( $key !~ /\./) { + $key = "me.$key"; + } + if ( is_type( $type, 'string' ) && $format eq 'uuid' ) { $param_filter{ $key } = $value ; } From 041e471168f80acbb162bf3eae0d275e525ad9a8 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Tue, 30 May 2023 23:16:23 +0530 Subject: [PATCH 4/8] added Lingua::EN::Inflect::Phrase as dependency --- Makefile.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.PL b/Makefile.PL index f77e77f5..867bfc2a 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -33,6 +33,7 @@ my %WriteMakefileArgs = ( "File::Spec::Functions" => 0, "FindBin" => 0, "JSON::Validator" => "5.00", + "Lingua::EN::Inflect::Phrase" => 0, "Mojolicious" => 9, "Mojolicious::Plugin::I18N" => "1.6", "Mojolicious::Plugin::OpenAPI" => "5.00", From 2f8e202077d1c07e725cc998ab36ba319c4ec4b3 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Tue, 30 May 2023 23:20:40 +0530 Subject: [PATCH 5/8] added Lingua::EN::Inflect::Phrase as dependency in cpanfile --- Makefile.PL | 1 - cpanfile | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.PL b/Makefile.PL index 867bfc2a..f77e77f5 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -33,7 +33,6 @@ my %WriteMakefileArgs = ( "File::Spec::Functions" => 0, "FindBin" => 0, "JSON::Validator" => "5.00", - "Lingua::EN::Inflect::Phrase" => 0, "Mojolicious" => 9, "Mojolicious::Plugin::I18N" => "1.6", "Mojolicious::Plugin::OpenAPI" => "5.00", diff --git a/cpanfile b/cpanfile index 4f5f6f9a..8012599b 100644 --- a/cpanfile +++ b/cpanfile @@ -3,6 +3,7 @@ requires "Digest" => "0"; requires "Exporter" => "0"; requires "File::Spec::Functions" => "0"; requires "FindBin" => "0"; +requires "Lingua::EN::Inflect::Phrase" => "0"; requires "JSON::Validator" => "5.00"; requires "Mojolicious" => "9"; requires "Mojolicious::Plugin::I18N" => "1.6"; From 94ac1ad32edb4876b35c90b77137b74ffc3a8932 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Thu, 1 Jun 2023 22:21:25 +0530 Subject: [PATCH 6/8] add 'me.' before relation only in case of joins --- lib/Yancy/Controller/Yancy.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Yancy/Controller/Yancy.pm b/lib/Yancy/Controller/Yancy.pm index 5457cce7..4d9a0d6b 100644 --- a/lib/Yancy/Controller/Yancy.pm +++ b/lib/Yancy/Controller/Yancy.pm @@ -1298,7 +1298,7 @@ sub _get_list_args { my $value = $c->param( $key ); # if the key belongs to main relation prefix with 'me.' - if ( $key !~ /\./) { + if ($opt->{ join } && $key !~ /\./) { $key = "me.$key"; } From 90468d33c5feef7022f8f20231468ab7f4e8b5b2 Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Thu, 1 Jun 2023 23:07:12 +0530 Subject: [PATCH 7/8] security fix: prevent parameter provided joins to bypass stash provided joins --- lib/Yancy/Controller/Yancy.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/Yancy/Controller/Yancy.pm b/lib/Yancy/Controller/Yancy.pm index 4d9a0d6b..df7e9093 100644 --- a/lib/Yancy/Controller/Yancy.pm +++ b/lib/Yancy/Controller/Yancy.pm @@ -1263,15 +1263,16 @@ sub _get_list_args { $opt->{order_by} = $order_by; } - my $joins = $c->every_param('join'); - if ( $joins && @$joins ) { - $opt->{ join } = $joins; + if ( my $join = $c->param( 'join' ) ) { + $opt->{ join } = $join; } if ( my $join = $c->stash( 'join' ) ) { $opt->{ join } = $join; } + my $joins = $opt->{join} ? [ $opt-> { join } ] : [] ; + my $schema = $c->schema; my $model = $schema->model; From aa730022386f5ec9d99409929f6a367fca2aa4de Mon Sep 17 00:00:00 2001 From: Rajesh Mallah Date: Fri, 2 Jun 2023 00:44:18 +0530 Subject: [PATCH 8/8] support for complex joins on list method via json encoding. support for specifying complex joins via query parameter is being introduced. The json is encoded as per uri encoding scheme and is decoded into a perl reference. The same reference is passed as the 'join' parameter in DBIC. This allows the same familier syntax to be adopted for specifying the joins. How it works: * The data structure in join is passed to a function _get_unique_relationships to get all the unique relations as keys of a hash. for each of such relation the properties are determined and are stored in another hash having keys in the format 'relation_name.key_name' , the value being the properties of the column. these properties help in using the appropriate operators at sql level. Note that it is also possible to filter the results based on any of the relationship columns by specifying as relation.columnname=somevalue in query parameters. --- lib/Yancy/Controller/Yancy.pm | 39 +++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/lib/Yancy/Controller/Yancy.pm b/lib/Yancy/Controller/Yancy.pm index df7e9093..eb21377d 100644 --- a/lib/Yancy/Controller/Yancy.pm +++ b/lib/Yancy/Controller/Yancy.pm @@ -207,7 +207,7 @@ L =cut use Mojo::Base 'Mojolicious::Controller'; -use Mojo::JSON qw( to_json ); +use Mojo::JSON qw( to_json from_json ); use Yancy::Util qw( derp is_type ); use POSIX qw( ceil ); @@ -1265,13 +1265,29 @@ sub _get_list_args { if ( my $join = $c->param( 'join' ) ) { $opt->{ join } = $join; + my $json_join_param = eval { from_json ( $join ) }; + if ( $json_join_param ) { + $opt-> { join } = $json_join_param ; + } } if ( my $join = $c->stash( 'join' ) ) { $opt->{ join } = $join; } - my $joins = $opt->{join} ? [ $opt-> { join } ] : [] ; + # use Data::Dumper; + + my $joins = []; + if ( $opt->{ join } ) { + # $c->app->log->info( sprintf "join is: %s" , Dumper $opt->{ join } ); + my %joins ; + _get_unique_relations ( $opt-> { join } , \%joins ); + if ( keys %joins) { + $joins = [ keys %joins ]; + } + } + + # $c->app->log->info( sprintf "joins is: %s" , Dumper $joins ); my $schema = $c->schema; my $model = $schema->model; @@ -1345,6 +1361,25 @@ sub _get_list_args { return ( $filter, $opt ); } +sub _get_unique_relations { + my ( $obj, $result) = @_; + unless ( ref $obj) { + $result->{$obj}++; + return; + } + if ( ref $obj eq 'ARRAY' && @$obj > 0 ) { + foreach my $x ( @$obj) { + &_get_unique_relations ( $x , $result); + } + } + if ( ref $obj eq 'HASH' && keys %{$obj} > 0 ) { + foreach my $key ( keys %{$obj} ) { + &_get_unique_relations ( $key , $result); + &_get_unique_relations ( $obj->{$key} , $result); + } + } +} + sub _resolve_filter { my ( $c ) = @_; my $filter = $c->stash( 'filter' );