From 0e11bcda1713288ead74231bacd033b9299d20a1 Mon Sep 17 00:00:00 2001 From: osaajani <> Date: Fri, 20 Dec 2024 23:32:56 +0100 Subject: [PATCH] Add strategy for more efficient pagination in API listing of entries by using before_id and after_id wheres instead of pagination if possible --- controllers/internals/Queue.php | 2 +- controllers/internals/Received.php | 2 +- controllers/internals/StandardController.php | 6 ++- controllers/publics/Api.php | 17 ++++--- models/Call.php | 43 +++++++++++++++-- models/Group.php | 49 +++++++++++++++++--- models/PhoneGroup.php | 47 +++++++++++++++++-- models/RedisQueue.php | 2 +- models/StandardModel.php | 14 +++++- routes.php | 2 + 10 files changed, 156 insertions(+), 28 deletions(-) diff --git a/controllers/internals/Queue.php b/controllers/internals/Queue.php index 2275cb9..451e5dd 100644 --- a/controllers/internals/Queue.php +++ b/controllers/internals/Queue.php @@ -66,7 +66,7 @@ class Queue extends \descartes\InternalController * * @return mixed $message : The oldest message or null if no message found, can be anything * @param ?string $tag : A tag to associate to the message for routing purposes, if null will read from general queue - * @param mixed : The message to add to the queue, can be anything, the queue will have to treat it by itself + * @param mixed : The message return from the queue, can be anything, null if no message found */ public function read(?string $tag = null) { diff --git a/controllers/internals/Received.php b/controllers/internals/Received.php index 6bace82..320a62d 100644 --- a/controllers/internals/Received.php +++ b/controllers/internals/Received.php @@ -83,7 +83,7 @@ use Exception; $this->bdd->beginTransaction(); $id_received = $this->get_model()->insert($received); - if (!$id_received) + if (!$id_received) { $this->bdd->rollBack(); diff --git a/controllers/internals/StandardController.php b/controllers/internals/StandardController.php index 7018244..3d296cc 100644 --- a/controllers/internals/StandardController.php +++ b/controllers/internals/StandardController.php @@ -77,12 +77,14 @@ namespace controllers\internals; * @param int $id_user : User id * @param ?int $nb_entry : Number of entry to return * @param ?int $page : Pagination, used to calcul offset, $nb_entry * $page + * @param ?int $after_id : If provided use where id > $after_id instead of offset + * @param ?int $before_id : If provided use where id < $before_id instead of offset * * @return array : Entrys list */ - public function list_for_user(int $id_user, ?int $nb_entry = null, ?int $page = null) + public function list_for_user(int $id_user, ?int $nb_entry = null, ?int $page = null, ?int $after_id = null, ?int $before_id = null) { - return $this->get_model()->list_for_user($id_user, $nb_entry, $nb_entry * $page); + return $this->get_model()->list_for_user($id_user, $nb_entry, $nb_entry * $page, $after_id, $before_id); } /** diff --git a/controllers/publics/Api.php b/controllers/publics/Api.php index 6c25da5..ae40c7f 100644 --- a/controllers/publics/Api.php +++ b/controllers/publics/Api.php @@ -126,10 +126,13 @@ namespace controllers\publics; * * @param string $entry_type : Type of entries we want to list ['sended', 'received', 'scheduled', 'contact', 'group', 'conditional_group', 'phone', 'phone_group', 'media'] * @param int $page : Pagination number, Default = 0. Group of 25 results. + * @param ?int $after_id : If provided use where id > $after_id instead of offset based on page, more performant + * @param ?int $before_id : If provided use where id < $before_id instead of offset based on page, more performant + * * * @return : List of entries */ - public function get_entries(string $entry_type, int $page = 0) + public function get_entries(string $entry_type, int $page = 0, ?int $after_id = null, ?int $before_id = null) { $entry_types = ['sended', 'received', 'scheduled', 'contact', 'group', 'conditional_group', 'phone', 'phone_group', 'media']; @@ -148,7 +151,7 @@ namespace controllers\publics; $page = (int) $page; $limit = 25; - $entries = $controller->list_for_user($this->user['id'], $limit, $page); + $entries = $controller->list_for_user($this->user['id'], $limit, $page, $after_id, $before_id); //Special case for scheduled, we must add numbers because its a join if ('scheduled' === $entry_type) @@ -221,14 +224,16 @@ namespace controllers\publics; $return = self::DEFAULT_RETURN; $return['response'] = $entries; - if (\count($entries) === $limit) + if (\count($entries) === $limit || ($entries && $before_id)) { - $return['next'] = \descartes\Router::url('Api', __FUNCTION__, ['entry_type' => $entry_type, 'page' => $page + 1], ['api_key' => $this->user['api_key']]); + $last_entry = end($entries); + $return['next'] = \descartes\Router::url('Api', __FUNCTION__, ['entry_type' => $entry_type, 'after_id' => $last_entry['id']], ['api_key' => $this->user['api_key']]); } - if ($page > 0) + if ($page > 0 || ($entries && ($after_id || $before_id))) { - $return['prev'] = \descartes\Router::url('Api', __FUNCTION__, ['entry_type' => $entry_type, 'page' => $page - 1], ['api_key' => $this->user['api_key']]); + $first_entry = $entries[0]; + $return['prev'] = \descartes\Router::url('Api', __FUNCTION__, ['entry_type' => $entry_type, 'before_id' => $first_entry['id']], ['api_key' => $this->user['api_key']]); } $this->auto_http_code(true); diff --git a/models/Call.php b/models/Call.php index a09395d..82eab5f 100644 --- a/models/Call.php +++ b/models/Call.php @@ -26,10 +26,13 @@ namespace models; * @param int $id_user : user id * @param ?int $limit : Number of entry to return or null * @param ?int $offset : Number of entry to ignore or null + * @param ?int $after_id : If provided use where id > $after_id instead of offset + * @param ?int $before_id : If provided use where id < $before_id instead of offset + * * * @return array */ - public function list_for_user(int $id_user, $limit, $offset) + public function list_for_user(int $id_user, $limit, $offset, ?int $after_id = null, ?int $before_id = null) { $query = ' SELECT `call`.*, contact.name as contact_name, phone.name as phone_name @@ -42,6 +45,37 @@ namespace models; WHERE `call`.id_user = :id_user '; + $params = [ + 'id_user' => $id_user, + ]; + + if ($after_id || $before_id) + { + $offset = null; + } + + if ($after_id) + { + $query .= ' + AND `call`.id > :after_id + ORDER BY `call`.id + '; + $params['after_id'] = $after_id; + } + elseif ($before_id) + { + $query .= ' + AND `call`.id < :before_id + ORDER BY `call`.id DESC + '; + $params['before_id'] = $before_id; + } + else + { + $query .= ' ORDER BY `call`.id'; + } + + if (null !== $limit) { $limit = (int) $limit; @@ -54,9 +88,10 @@ namespace models; } } - $params = [ - 'id_user' => $id_user, - ]; + if ($before_id) + { + return array_reverse($this->_run_query($query, $params)); + } return $this->_run_query($query, $params); } diff --git a/models/Group.php b/models/Group.php index d247b93..a30a853 100644 --- a/models/Group.php +++ b/models/Group.php @@ -20,10 +20,12 @@ namespace models; * @param int $id_user : user id * @param ?int $limit : Number of entry to return or null * @param ?int $offset : Number of entry to ignore or null + * @param ?int $after_id : If provided use where id > $after_id instead of offset + * @param ?int $before_id : If provided use where id < $before_id instead of offset * * @return array */ - public function list_for_user(int $id_user, $limit, $offset) + public function list_for_user(int $id_user, $limit, $offset, ?int $after_id = null, ?int $before_id = null) { $query = ' SELECT g.*, COUNT(gc.id) as nb_contact @@ -31,9 +33,43 @@ namespace models; LEFT JOIN group_contact as gc ON g.id = gc.id_group WHERE id_user = :id_user - GROUP BY g.id '; + $params = [ + 'id_user' => $id_user, + ]; + + if ($after_id || $before_id) + { + $offset = null; + } + + if ($after_id) + { + $query .= ' + AND g.id > :after_id + GROUP BY g.id + ORDER BY g.id + '; + $params['after_id'] = $after_id; + } + elseif ($before_id) + { + $query .= ' + AND g.id < :before_id + GROUP BY g.id + ORDER BY g.id DESC + '; + $params['before_id'] = $before_id; + } + else + { + $query .= ' + GROUP BY g.id + ORDER BY g.id + '; + } + if (null !== $limit) { $limit = (int) $limit; @@ -46,10 +82,11 @@ namespace models; } } - $params = [ - 'id_user' => $id_user, - ]; - + if ($before_id) + { + return array_reverse($this->_run_query($query, $params)); + } + return $this->_run_query($query, $params); } diff --git a/models/PhoneGroup.php b/models/PhoneGroup.php index 0a71a24..3f31d49 100644 --- a/models/PhoneGroup.php +++ b/models/PhoneGroup.php @@ -20,10 +20,12 @@ namespace models; * @param int $id_user : user id * @param ?int $limit : Number of entry to return or null * @param ?int $offset : Number of entry to ignore or null + * @param ?int $after_id : If provided use where id > $after_id instead of offset + * @param ?int $before_id : If provided use where id < $before_id instead of offset * * @return array */ - public function list_for_user(int $id_user, $limit, $offset) + public function list_for_user(int $id_user, $limit, $offset, ?int $after_id = null, ?int $before_id = null) { $query = ' SELECT pg.*, COUNT(pgp.id) as nb_phone @@ -31,9 +33,43 @@ namespace models; LEFT JOIN phone_group_phone as pgp ON pg.id = pgp.id_phone_group WHERE pg.id_user = :id_user - GROUP BY pg.id '; + $params = [ + 'id_user' => $id_user, + ]; + + if ($after_id || $before_id) + { + $offset = null; + } + + if ($after_id) + { + $query .= ' + AND pg.id > :after_id + GROUP BY pg.id + ORDER BY g.id + '; + $params['after_id'] = $after_id; + } + elseif ($before_id) + { + $query .= ' + AND pg.id < :before_id + GROUP BY pg.id + ORDER BY g.id DESC + '; + $params['before_'] = $before_id; + } + else + { + $query .= ' + GROUP BY pg.id + ORDER BY pg.id + '; + } + if (null !== $limit) { $limit = (int) $limit; @@ -46,9 +82,10 @@ namespace models; } } - $params = [ - 'id_user' => $id_user, - ]; + if ($before_id) + { + return array_reverse($this->_run_query($query, $params)); + } return $this->_run_query($query, $params); } diff --git a/models/RedisQueue.php b/models/RedisQueue.php index 95ac92d..8c11833 100644 --- a/models/RedisQueue.php +++ b/models/RedisQueue.php @@ -70,7 +70,7 @@ class RedisQueue implements Queue * * @return mixed $message : The oldest message or null if no message found, can be anything * @param ?string $tag : A tag to associate to the message for routing purposes, if null will read from general queue - * @param mixed : The message to add to the queue, can be anything, the queue will have to treat it by itself + * @param mixed : The message return from the queue, can be anything, null if no message found */ public function read(?string $tag = null) { diff --git a/models/StandardModel.php b/models/StandardModel.php index 45bf967..99a7c1b 100644 --- a/models/StandardModel.php +++ b/models/StandardModel.php @@ -70,12 +70,22 @@ namespace models; * @param int $id_user : user id * @param int $limit : Number of entry to return * @param int $offset : Number of entry to ignore + * @param ?int $after_id : If provided use where id > $after_id instead of offset + * @param ?int $before_id : If provided use where id < $before_id instead of offset * * @return array */ - public function list_for_user(int $id_user, $limit, $offset) + public function list_for_user(int $id_user, $limit, $offset, ?int $after_id = null, ?int $before_id = null) { - return $this->_select($this->get_table_name(), ['id_user' => $id_user], null, false, $limit, $offset); + if ($after_id !== null) { + return $this->_select($this->get_table_name(), ['id_user' => $id_user, '>id' => $after_id], 'id', false, $limit); + } + + if ($before_id !== null) { + return array_reverse($this->_select($this->get_table_name(), ['id_user' => $id_user, ' $before_id], 'id', true, $limit)); + } + + return $this->_select($this->get_table_name(), ['id_user' => $id_user], 'id', false, $limit, $offset); } /** diff --git a/routes.php b/routes.php index bff3f98..6cfe1ca 100644 --- a/routes.php +++ b/routes.php @@ -214,6 +214,8 @@ 'get_entries' => [ '/api/list/{entry_type}/', '/api/list/{entry_type}/{page}/', + '/api/list/{entry_type}/after/{after_id}/', + '/api/list/{entry_type}/before/{before_id}/', ], 'get_usage' => '/api/usage/', 'get_sms_status_stats' => '/api/stats/sms-status/',