v3.7.0 add stats about sended sms status, add better support for haproxy, improve gammu ream sms with deletion of read sms

Improve Gammu SMS Receiver

VERSION

@@ -1 +1 @@
-v3.6.4
+v3.7.0

adapters/GammuAdapter.php

@@ -280,6 +280,7 @@ namespace adapters;
             $command_parts = [
                 PWD . '/bin/gammu_get_unread_sms.py',
                 escapeshellarg($this->data['config_file']),
+                '--delete'
             ];
 
             $return = $this->exec_command($command_parts);

bin/gammu_get_unread_sms.py

@@ -1,56 +1,236 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
 # vim: expandtab sw=4 ts=4 sts=4:
-#
+
+# (C) 2003 - 2018 Michal Čihař <michal@cihar.com> - python-gammu
+# (C) 2015 - 2021 Raspian France <raspbianfrance@gmail.com> - RaspianFrance/raspisms
+# (C) 2022 - Orsiris de Jong <orsiris.dejong@netperfect.fr> - NetInvent SASU
+
 
 from __future__ import print_function
-import gammu
-import sys
-import json
 
-def main():
+
+__intname__ = "gammu_get_unread_sms.py"
+__author__ = "Orsiris de Jong - <orsiris.dejong@netperfect.fr>"
+__version__ = "2.0.2"
+__build__ = "2022102501"
+__compat__ = "python2.7+"
+
+
+import os
+import gammu
+import json
+import logging
+from logging.handlers import RotatingFileHandler
+import tempfile
+from argparse import ArgumentParser
+import subprocess
+import re
+
+LOG_FILE = "/var/log/{}.log".format(__intname__)
+_DEBUG = os.environ.get("_DEBUG", False)
+logger = logging.getLogger(__name__)
+
+
+def get_logger(log_file):
+    # We would normally use ofunctions.logger_utils here with logger_get_logger(), but let's keep no dependencies
+    try:
+        try:
+            filehandler = RotatingFileHandler(
+                log_file, mode="a", encoding="utf-8", maxBytes=1048576, backupCount=3
+            )
+        except OSError:
+            try:
+                temp_log_file = tempfile.gettempdir() + os.sep + __name__ + ".log"
+                filehandler = RotatingFileHandler(
+                    temp_log_file,
+                    mode="a",
+                    encoding="utf-8",
+                    maxBytes=1048576,
+                    backupCount=3,
+                )
+            except OSError as exc:
+                print("Cannot create log file: %s" % exc.__str__())
+                filehandler = None
+
+        _logger = logging.getLogger()
+        if _DEBUG:
+            _logger.setLevel(logging.DEBUG)
+        else:
+            _logger.setLevel(logging.INFO)
+
+        formatter = logging.Formatter("%(asctime)s :: %(levelname)s :: %(message)s")
+        if filehandler:
+            filehandler.setFormatter(formatter)
+            _logger.addHandler(filehandler)
+        consolehandler = logging.StreamHandler()
+        consolehandler.setFormatter(formatter)
+        _logger.addHandler(consolehandler)
+        return _logger
+    except Exception as exc:
+        print("Cannot create logger instance: %s" % exc.__str__())
+
+
+def get_gammu_version():
+    # Quite badly coded, i'd use command_runner but I try to not have dependencies here
+    try:
+        proc = subprocess.Popen(
+            ["LC_ALL=C gammu", "--version"], shell=True, stdout=subprocess.PIPE
+        )
+        stdout, _ = proc.communicate()
+        version = re.search(r"Gammu version ([0-9]+)\.([0-9]+)\.([0-9]+)", str(stdout))
+        # dont' bother to return version[0] since it's the whole match
+        return (int(version[1]), int(version[2]), int(version[3]))
+    except Exception as exc:
+        logger.error("Cannot get gammu version: %s" % exc.__str__())
+        return None
+
+
+def get_gammu_handle(config_file):
     state_machine = gammu.StateMachine()
 
-    if len(sys.argv) < 2:
+    if config_file:
-        sys.exit(1)
+        state_machine.ReadConfig(Filename=config_file)
     else:
-        state_machine.ReadConfig(Filename=sys.argv[1])
+        state_machine.Readconfig()
-        del sys.argv[1]
-
     state_machine.Init()
 
+    return state_machine
+
+
+def load_sms_from_gammu(state_machine):
+    """
+    The actual function that retrieves SMS via GAMMU from your modem / phone
+    Also concatenates multiple SMS into single long SMS
+    """
     status = state_machine.GetSMSStatus()
 
-    remain = status['SIMUsed'] + status['PhoneUsed'] + status['TemplatesUsed']
+    remaining_sms = status["SIMUsed"] + status["PhoneUsed"] + status["TemplatesUsed"]
-
+    logger.debug("Found %s sms" % remaining_sms)
-    start = True
+    sms_list = []
 
     try:
-        while remain > 0:
+        is_first_message = True
-            if start:
+        while remaining_sms > 0:
-                sms = state_machine.GetNextSMS(Start=True, Folder=0)
+            if is_first_message:
-                start = False
+                sms = state_machine.GetNextSMS(Start=is_first_message, Folder=0)
+                is_first_message = False
             else:
-                sms = state_machine.GetNextSMS(
+                sms = state_machine.GetNextSMS(Location=sms[0]["Location"], Folder=0)
-                    Location=sms[0]['Location'], Folder=0
+            remaining_sms = remaining_sms - len(sms)
-                )
+            sms_list.append(sms)
-            remain = remain - len(sms)
-
-            for m in sms :
-                if m['State'] != 'UnRead' :
-                    continue
-
-                print(json.dumps({
-                    'number': m['Number'],
-                    'at': str(m['DateTime']),
-                    'status': m['State'],
-                    'text': m['Text'],
-                }))
 
     except gammu.ERR_EMPTY:
-        #do noting
+        logger.debug("Finished reading all messages")
-        return True
+
+    # Concat multiple SMS into list of sms that go together using LinkSMS
+    return gammu.LinkSMS(sms_list)
 
 
-if __name__ == '__main__':
+def render_sms_as_json(state_machine, sms_list, delete_sms, show_read_sms):
-    main()
+    """
+    Provided sms_list is a list of lists of sms, eg
+    sms_list = [
+                   [sms],
+                   [sms1, sms2],  #  When two sms are in the same list, they form a long sms
+                   [sms],
+               ]
+
+    Concatenate long SMS from multiple sends and print them as JSON on stdout
+    """
+
+    for sms in sms_list:
+        if sms[0]["State"] == "UnRead" or show_read_sms:
+            sms_text = ""
+            for to_concat_sms in sms:
+                sms_text += to_concat_sms["Text"]
+            print(
+                json.dumps(
+                    {
+                        "number": sms[0]["Number"],
+                        "at": str(sms[0]["DateTime"]),
+                        "status": sms[0]["State"],
+                        "text": sms_text,
+                    }
+                )
+            )
+
+            if delete_sms:
+                for to_concat_sms in sms:
+                    try:
+                        state_machine.DeleteSMS(
+                            to_concat_sms["Folder"], to_concat_sms["Location"]
+                        )
+                    except Exception as exc:
+                        logger.error("Cannot delete sms: %s" % exc.__str__())
+
+
+def main(config_file, delete_sms, show_read):
+    # type: (bool, bool) -> None
+    logger.debug("Running gammu receiver with config {}".format(config_file))
+
+    try:
+        # Mandatory modem config file
+        # config_file = sys.argv[1]
+
+        state_machine = get_gammu_handle(config_file)
+
+        sms_list = load_sms_from_gammu(state_machine)
+        render_sms_as_json(state_machine, sms_list, delete_sms, show_read)
+
+    except Exception as exc:
+        logger.error("Could not retrieve SMS from Gammu: %s" % exc.__str__())
+        logger.debug("Trace:", exc_info=True)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser("Gammu SMS retriever")
+    parser.add_argument(
+        "gammu_config_file", type=str, nargs="?", help="Gammu config file"
+    )
+    parser.add_argument("--debug", action="store_true", help="Activate debugging")
+    parser.add_argument(
+        "-l",
+        "--log-file",
+        type=str,
+        dest="log_file",
+        default=None,
+        help="Optional path to log file, defaults to /var/log",
+    )
+    parser.add_argument(
+        "--delete", action="store_true", help="Delete messages after they've been read"
+    )
+    parser.add_argument(
+        "--show-read", action="store_true", help="Also show already read messages"
+    )
+
+    args = parser.parse_args()
+
+    config_file = args.gammu_config_file
+
+    if args.log_file:
+        LOG_FILE = args.log_file
+
+    if args.debug:
+        _DEBUG = args.debug
+
+    _logger = get_logger(LOG_FILE)
+    if _logger:
+        logger = _logger
+
+    delete = False
+    if args.delete:
+        # We need to check if we have gammu >= 1.42.0 since deleting sms with lower versions fail with:
+        # Cannot delete sms: {'Text': 'The type of memory is not available or has been disabled.', 'Where': 'DeleteSMS', 'Code': 81}
+        # see https://github.com/gammu/gammu/issues/460
+        try:
+            gammu_version = get_gammu_version()
+            if gammu_version[0] > 1 or (gammu_version[0] == 1 and gammu_version[1] >= 42):
+                delete = True
+            else:
+                logger.warning("Cannot delete SMS. You need gammu >= 1.42.0.")
+        except TypeError:
+            logger.warning("Cannot get gammu version. SMS Deleting might not work properly.")
+
+    show_read = args.show_read
+    main(config_file, delete, show_read)

controllers/internals/Console.php

@@ -11,6 +11,8 @@
 
 namespace controllers\internals;
 
+use DateInterval;
+
     /**
      * Class to call the console scripts.
      */
@@ -212,4 +214,46 @@ namespace controllers\internals;
             $internal_quota = new \controllers\internals\Quota($bdd);
             $internal_quota->renew_quotas();
         }
+
+        /**
+         * Do some fake population renewal.
+         */
+        public function f()
+        {
+            $bdd = \descartes\Model::_connect(DATABASE_HOST, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD);
+            $internal_sended = new \controllers\internals\Sended($bdd);
+
+            $destinations = ['+33612345678','+33612345679','+33612345680',];
+            $statuses = [\models\Sended::STATUS_DELIVERED, \models\Sended::STATUS_FAILED, \models\Sended::STATUS_UNKNOWN];
+            $day = new \DateTime();
+            $day->sub(new DateInterval('P30D'));
+            for ($i = 0; $i < 30; $i++)
+            {
+                $day->add(new DateInterval('P1D'));
+                $n = rand(0, 100);
+                for ($j = 0; $j < $n; $j++)
+                {
+                    $id_user = 1;
+                    $id_phone = rand(1, 2);
+                    $destination = $destinations[array_rand($destinations)];
+                    $status = $statuses[array_rand($statuses)];
+                    $internal_sended->create(
+                        $id_user, 
+                        $id_phone, 
+                        $day->format('Y-m-d H:i:s'), 
+                        "TEST N°$i:$j", 
+                        $destination, 
+                        uniqid(), 
+                        'adapters\TestAdapter', 
+                        false,
+                        false,
+                        null,
+                        [],
+                        null,
+                        $status,
+                    );
+                }
+            }
+            
+        }
     }

controllers/internals/Sended.php

@@ -207,17 +207,9 @@ use Exception;
          *
          * @return array
          */
-        public function count_by_day_since_for_user(int $id_user, $date)
+        public function count_by_day_and_status_since_for_user(int $id_user, $date)
         {
-            $counts_by_day = $this->get_model()->count_by_day_since_for_user($id_user, $date);
+            return $this->get_model()->count_by_day_and_status_since_for_user($id_user, $date);
-            $return = [];
-
-            foreach ($counts_by_day as $count_by_day)
-            {
-                $return[$count_by_day['at_ymd']] = $count_by_day['nb'];
-            }
-
-            return $return;
         }
 
         /**

controllers/publics/Dashboard.php

@@ -87,11 +87,12 @@ namespace controllers\publics;
                 $stats_start_date_formated = $stats_start_date->format('Y-m-d');
             }
 
-            $nb_sendeds_by_day = $this->internal_sended->count_by_day_since_for_user($id_user, $stats_start_date_formated);
+            $nb_sendeds_by_day = $this->internal_sended->count_by_day_and_status_since_for_user($id_user, $stats_start_date_formated);
             $nb_receiveds_by_day = $this->internal_received->count_by_day_since_for_user($id_user, $stats_start_date_formated);
 
             //On va traduire ces données pour les afficher en graphique
-            $array_area_chart = [];
+            $array_bar_chart_sended = [];
+            $array_bar_chart_received = [];
 
             $date = clone $stats_start_date;
             $one_day = new \DateInterval('P1D');
@@ -101,12 +102,15 @@ namespace controllers\publics;
             while ($date <= $now)
             {
                 $date_f = $date->format('Y-m-d');
-                $array_area_chart[$date_f] = [
+                $array_bar_chart_sended[$date_f] = [
                     'period' => $date_f,
-                    'sendeds' => 0,
+                    'sendeds_failed' => 0,
-                    'receiveds' => 0,
+                    'sendeds_unknown' => 0,
+                    'sendeds_delivered' => 0,
                 ];
 
+                $array_bar_chart_received[$date_f] = ['period' => $date_f, 'receiveds' => 0];
+
                 $date->add($one_day);
             }
 
@@ -114,15 +118,16 @@ namespace controllers\publics;
             $total_receiveds = 0;
 
             //0n remplie le tableau avec les données adaptées
-            foreach ($nb_sendeds_by_day as $date => $nb_sended)
+            foreach ($nb_sendeds_by_day as $nb_sended)
             {
-                $array_area_chart[$date]['sendeds'] = $nb_sended;
+                $array_bar_chart_sended[$nb_sended['at_ymd']]['sendeds_' . $nb_sended['status']] = $nb_sended['nb'];
-                $total_sendeds += $nb_sended;
+                $array_bar_chart_sended[$nb_sended['at_ymd']]['sendeds_total'] = ($array_bar_chart_sended[$nb_sended['at_ymd']]['sendeds_total'] ?? 0) + $nb_sended['nb'];
+                $total_sendeds += $nb_sended['nb'];
             }
 
             foreach ($nb_receiveds_by_day as $date => $nb_received)
             {
-                $array_area_chart[$date]['receiveds'] = $nb_received;
+                $array_bar_chart_received[$date]['receiveds'] = $nb_received;
                 $total_receiveds += $nb_received;
             }
 
@@ -130,7 +135,8 @@ namespace controllers\publics;
             $avg_sendeds = round($total_sendeds / $nb_days, 2);
             $avg_receiveds = round($total_receiveds / $nb_days, 2);
 
-            $array_area_chart = array_values($array_area_chart);
+            $array_bar_chart_sended = array_values($array_bar_chart_sended);
+            $array_bar_chart_received = array_values($array_bar_chart_received);
 
             $this->render('dashboard/show', [
                 'nb_contacts' => $nb_contacts,
@@ -145,7 +151,9 @@ namespace controllers\publics;
                 'sendeds' => $sendeds,
                 'receiveds' => $receiveds,
                 'events' => $events,
-                'data_area_chart' => json_encode($array_area_chart),
+                'data_bar_chart_sended' => json_encode($array_bar_chart_sended),
+                'data_bar_chart_received' => json_encode($array_bar_chart_received),
+                'stats_start_date_formated' => $stats_start_date_formated,
             ]);
         }
     }

models/Sended.php

@@ -222,14 +222,14 @@ namespace models;
          *
          * @return array
          */
-        public function count_by_day_since_for_user($id_user, $date)
+        public function count_by_day_and_status_since_for_user($id_user, $date)
         {
             $query = "
-                SELECT COUNT(id) as nb, DATE_FORMAT(at, '%Y-%m-%d') as at_ymd
+                SELECT COUNT(id) as nb, status, DATE_FORMAT(at, '%Y-%m-%d') as at_ymd
                 FROM sended
                 WHERE at > :date
                 AND id_user = :id_user
-                GROUP BY at_ymd
+                GROUP BY at_ymd, status
             ";
 
             $params = [

templates/dashboard/show.php

@@ -120,16 +120,29 @@
                 <div class="col-lg-12">
                     <div class="panel panel-default dashboard-panel-chart">
                         <div class="panel-heading">
-							<h3 class="panel-title"><i class="fa fa-area-chart fa-fw"></i> Activité de la semaine : </h3>
+                            <h3 class="panel-title"><i class="fa fa-area-chart fa-fw"></i> SMS envoyés depuis le <?= $stats_start_date_formated; ?> : </h3>
                             <span style="color: #5CB85C;">SMS envoyés (moyenne = <?php echo $avg_sendeds; ?> par jour).</span><br/>
-							<span style="color: #EDAB4D">SMS reçus (moyenne = <?php echo $avg_receiveds; ?> par jour).</span>
                             <?php if ($quota_unused) { ?>
                                 <br/>
                                 <span style="color: #d9534f">Crédits restants : <?= $quota_unused; ?>.</span>
                             <?php } ?>
                         </div>
                         <div class="panel-body">
-							<div id="morris-area-chart"></div>
+                            <div id="morris-bar-chart-sended"></div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+			<div class="row">
+                <div class="col-lg-12">
+                    <div class="panel panel-default dashboard-panel-chart">
+                        <div class="panel-heading">
+                            <h3 class="panel-title"><i class="fa fa-area-chart fa-fw"></i> SMS reçus depuis le <?= $stats_start_date_formated; ?> : </h3>
+                            <span style="color: #EDAB4D">SMS reçus (moyenne = <?php echo $avg_receiveds; ?> par jour).</span>
+                        </div>
+                        <div class="panel-body">
+							<div id="morris-bar-chart-received"></div>
                         </div>
                     </div>
                 </div>
@@ -244,22 +257,54 @@
 <script>
     jQuery(document).ready(function()
     {
-		Morris.Area({
+        Morris.Bar({
-			element: 'morris-area-chart',
+            element: 'morris-bar-chart-sended',
-			behaveLikeLine: true,
             fillOpacity: 0.4,
-			data: <?php echo $data_area_chart;?>,
+            data: <?php echo $data_bar_chart_sended;?>,
             xkey: 'period',
             parseTime: false,
-			ykeys: ['sendeds', 'receiveds'],
+            ykeys: ['sendeds_failed', 'sendeds_unknown', 'sendeds_delivered'],
-			labels: ['SMS envoyés', 'SMS reçus'],
+            labels: ['SMS échoués', 'SMS inconnus', 'SMS délivrés'],
-			lineColors: ['#5CB85C', '#EDAB4D'],
+            barColors: ['#D9534F', '#337AB7', '#5CB85C'],
-            goals: [<?php echo $avg_sendeds; ?>, <?php echo $avg_receiveds; ?>],
+            goals: [<?php echo $avg_sendeds; ?>,],
-			goalLineColors: ['#5CB85C', '#EDAB4D'],
+            goalLineColors: ['#5CB85C'],
             goalStrokeWidth: 2,
             pointSize: 4,
             hideHover: 'auto',
-			resize: true
+            resize: true,
+            stacked: true,
+            hoverCallback: function (index, options, content, row) {
+				ret = '';
+				for (i = 0; i < options.ykeys.length; i++)
+				{
+					ret += options.labels[i];
+					ret += ' : ';
+					ret += row[options.ykeys[i]];
+					ret += ' (';
+					ret += (row[options.ykeys[i]] / (row.sendeds_total ? row.sendeds_total : 1) * 100).toFixed(2);
+					ret += '%)';
+					ret += "<br/>";
+				}
+
+				return ret;
+            }
+        });
+
+		Morris.Bar({
+            element: 'morris-bar-chart-received',
+            fillOpacity: 0.4,
+            data: <?php echo $data_bar_chart_received;?>,
+            xkey: 'period',
+            parseTime: false,
+            ykeys: ['receiveds'],
+            labels: ['SMS reçus'],
+            barColors: ['#EDAB4D'],
+            goals: [<?php echo $avg_receiveds; ?>],
+            goalLineColors: ['#EDAB4D'],
+            goalStrokeWidth: 2,
+            pointSize: 4,
+            hideHover: 'auto',
+            resize: true,
         });
     });
 </script>