Release 0.9.1-202504

- Install and update TimescaleDB Toolkit
- Remove deprecated client_id
- Remove index from logbook columns
- Remove deprecated column from api.logbook
- Add new mobilityDB support with comments
- Update api.export_logbook_geojson_linestring_trip_fn, add more metadata properties
- Update public.check_jwt, Make new mobilitydb export geojson function anonymous access
- Create api.monitoring_upsert_fn, the function that update api.metadata monitoring configuration
- Update cron_process_skplugin_upgrade_fn, update check for signalk plugin version
- Update api.find_log_from_moorage_fn using the mobilitydb trajectory
- Update api.find_log_to_moorage_fn using the mobilitydb trajectory
- Update api.eventlogs_view to fetch the events logs backwards and skip the new_stay
- Update api.stats_logs_fn, ensure the trip is completed
- Update api.stats_fn, ensure the trip is completed
- Update moorage_delete_trigger_fn, When morrage is deleted, delete process_queue references as well.
- Create public.logbook_delete_trigger_fn, When logbook is deleted, logbook_ext need to deleted as well.
- Update metadata table, mark client_id as deprecated
- Update public.process_lat_lon_fn, Add new moorage refrence in public.process_queue for event logs review
- Add Description on missing trigger
- Update public.logbook_active_geojson_fn, fix log_gis_line as there is no end time yet
- Create public.stay_active_geojson_fn function to produce a GeoJSON with the last position and stay details
- Update monitoring view to support live moorage in GeoJSON
- Add api.monitoring_live view, the live tracking view
- Refresh permissions

README.md

@@ -104,6 +104,8 @@ To understand the why and how, you might want to read [Why.md](https://github.co
 
 Remove the hassle of running PostgSail yourself. Here you can skip the technical setup, the maintenance work and server costs by getting PostgSail on our reliable and secure PostgSail Cloud. Register and try for free at [iot.openplotter.cloud](https://iot.openplotter.cloud/).
 
+PostgSail Cloud is Open Source and free for personal use with a single vessel. If wish to manage multiple boats contact us.
+
 ## On-Premise (for free)
 
 Self host postgSail where you want and how you want. There are no restrictions, you’re in full control. [Install Guide](https://github.com/xbgmsharp/postgsail/blob/main/docs/README.md)

docs/ERD/postgsail.md

@@ -26,11 +26,15 @@ erDiagram
         tfloat trip_batt_voltage "Battery Voltage"
         tfloat trip_cog "courseovergroundtrue"
         tfloat trip_depth "Depth"
+        tfloat trip_heading "heading True"
         tfloat trip_hum_out "Humidity outside"
         ttext trip_notes 
         tfloat trip_pres_out "Pressure outside"
         tfloat trip_sog "speedoverground"
+        tfloat trip_solar_power "solar powerPanel"
+        tfloat trip_solar_voltage "solar voltage"
         ttext trip_status 
+        tfloat trip_tank_level "Tank currentLevel"
         tfloat trip_temp_out "Temperature outside"
         tfloat trip_temp_water "Temperature water"
         tfloat trip_twa "windspeedapparent"
@@ -45,8 +49,6 @@ erDiagram
         jsonb available_keys "Signalk paths with unit for custom mapping"
         jsonb available_keys 
         double_precision beam 
-        text client_id "Deprecated client_id to be removed"
-        text client_id 
         jsonb configuration "Signalk path mapping for metrics"
         jsonb configuration 
         timestamp_with_time_zone created_at "{NOT_NULL}"

initdb/99_migrations_202412.sql

@@ -248,7 +248,7 @@ RETURNS TABLE (
 ) AS $$
 DECLARE
 BEGIN
-    -- Aggregate all metrics as trip ios short.
+    -- Aggregate all metrics as trip is short.
     RETURN QUERY
     WITH metrics AS (
         -- Extract metrics

initdb/99_migrations_202501.sql

@@ -17,19 +17,21 @@ select current_database();
 \echo 'Force timezone, just in case'
 set timezone to 'UTC';
 
--- Mark client_id as deprecated
+-- Update metadata table, mark client_id as deprecated
 COMMENT ON COLUMN api.metadata.client_id IS 'Deprecated client_id to be removed';
+-- Update metrics table, mark client_id as deprecated
 COMMENT ON COLUMN api.metrics.client_id IS 'Deprecated client_id to be removed';
 
--- Update metadata table COLUMN type to jsonb
+-- Update metadata table update configuration column type to jsonb and comment
 ALTER TABLE api.metadata ALTER COLUMN "configuration" TYPE jsonb USING "configuration"::jsonb;
 COMMENT ON COLUMN api.metadata.configuration IS 'Signalk path mapping for metrics';
 
--- Add new column available_keys
+-- Update metadata table add new column available_keys and comment
 ALTER TABLE api.metadata ADD available_keys jsonb NULL;
 COMMENT ON COLUMN api.metadata.available_keys IS 'Signalk paths with unit for custom mapping';
 
 --DROP FUNCTION public.metadata_upsert_trigger_fn();
+-- Update metadata_upsert_trigger_fn to metadata table to support configuration and available_keys and deprecated client_id
 CREATE OR REPLACE FUNCTION public.metadata_upsert_trigger_fn()
  RETURNS trigger
  LANGUAGE plpgsql
@@ -92,6 +94,7 @@ $function$
 COMMENT ON FUNCTION public.metadata_upsert_trigger_fn() IS 'process metadata from vessel, upsert';
 
 -- Create or replace the function that will be executed by the trigger
+-- Add metadata table trigger for update_metadata_configuration
 CREATE OR REPLACE FUNCTION api.update_metadata_configuration()
 RETURNS TRIGGER AS $$
 BEGIN
@@ -118,7 +121,7 @@ BEFORE UPDATE ON api.metadata
 FOR EACH ROW
 EXECUTE FUNCTION api.update_metadata_configuration();
 
--- Update api.export_logbook_geojson_linestring_trip_fn, add metadata
+-- Update api.export_logbook_geojson_linestring_trip_fn, add metadata properties
 CREATE OR REPLACE FUNCTION api.export_logbooks_geojson_linestring_trips_fn(
     start_log integer DEFAULT NULL::integer,
     end_log integer DEFAULT NULL::integer,

initdb/PGSAIL_VERSION

@@ -1 +1 @@
-0.9.0
+0.9.1

tests/index.js

@@ -636,14 +636,15 @@ request.set('User-Agent', 'PostgSail unit tests');
   describe("Vessel POST metrics, JWT vessel_role", function(){
 
     let data = [];
-    //console.log(vessel_metrics['metrics'][0]);
+    //console.log(test.vessel_metrics['metrics'][0]);
     let i;
     for (i = 0; i < test.vessel_metrics['metrics'].length; i++) {
       data[i] = test.vessel_metrics['metrics'][i];
       // Override time, -2h to allow to new data later without delay.
       data[i]['time'] = moment.utc().subtract(1, 'day').add(i, 'minutes').format();
       // Override client_id
-      data[i]['client_id'] = test.vessel_metadata.client_id;
+      //data[i]['client_id'] = test.vessel_metadata.client_id;
+      data[i]['client_id'] = null;
     }
     // The last entry are invalid and should be ignore.
     // - Invalid status

tests/index2.js

@@ -444,7 +444,8 @@ request.set('User-Agent', 'PostgSail unit tests');
       // Override time, +1h because previous sample include 47 entry.
       data[i]['time'] = moment.utc().subtract(2, 'hours').add(i, 'minutes').format();
       // Override client_id
-      data[i]['client_id'] = test.vessel_metadata.client_id;
+      //data[i]['client_id'] = test.vessel_metadata.client_id;
+      data[i]['client_id'] = null;
     }
     //console.log(data[0]);
 

tests/index4.js

@@ -163,6 +163,10 @@ var moment = require("moment");
         url: "/rpc/update_user_preferences_fn",
         payload: { key: "{public_monitoring}", value: true },
       },
+      {
+        url: "/rpc/update_user_preferences_fn",
+        payload: { key: "{public_timelapse}", value: true },
+      },
     ],
   },
   {
@@ -685,7 +689,7 @@ var moment = require("moment");
             let event = res.body;
             //console.log(event);
             // minimum events log for kapla & aava 13 + 4 email_otp = 17
-            event.length.should.be.aboveOrEqual(13);
+            event.length.should.be.aboveOrEqual(11);
             done(err);
           });
       });

tests/index5.js

@@ -151,7 +151,7 @@ var moment = require("moment");
     request.set("User-Agent", "PostgSail unit tests");
 
     describe("With no JWT as api_anonymous", function () {
-      it("/logs_view, api_anonymous no jwt token", function (done) {
+      it("/logs_view, api_anonymous no jwt token, x-is-public header", function (done) {
         // Reset agent so we do not save cookies
         request = supertest.agent(test.cname);
         request
@@ -167,7 +167,7 @@ var moment = require("moment");
             done(err);
           });
       });
-      it("/log_view, api_anonymous no jwt token", function (done) {
+      it("/log_view, api_anonymous no jwt token, x-is-public header", function (done) {
         // Reset agent so we do not save cookies
         request = supertest.agent(test.cname);
         request
@@ -183,7 +183,7 @@ var moment = require("moment");
             done(err);
           });
       });
-      it("/monitoring_view, api_anonymous no jwt token", function (done) {
+      it("/monitoring_view, api_anonymous no jwt token, x-is-public header", function (done) {
         // Reset agent so we do not save cookies
         request = supertest.agent(test.cname);
         request
@@ -200,7 +200,7 @@ var moment = require("moment");
             done(err);
           });
       });
-      it("/rpc/export_logbooks_geojson_linestring_trips_fn, api_anonymous no jwt token", function (done) {
+      it("/rpc/export_logbooks_geojson_linestring_trips_fn, api_anonymous no jwt token, x-is-public header", function (done) {
         // Reset agent so we do not save cookies
         request = supertest.agent(test.cname);
         request
@@ -214,10 +214,18 @@ var moment = require("moment");
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));
             res.header["server"].should.match(new RegExp("postgrest", "g"));
+            should.exist(res.body.geojson);
+            /*
+            if (res.body.geojson.features == null) { // aava
+              //res.body.geojson.features.should.not.be.ok();
+              done(err);
+            }
+            res.body.geojson.features.length.should.be.equal(4);
+            */
             done(err);
           });
       });
-      it("/rpc/export_logbooks_geojson_point_trips_fn, api_anonymous no jwt token", function (done) {
+      it("/rpc/export_logbooks_geojson_point_trips_fn, api_anonymous no jwt token, x-is-public header", function (done) {
         // Reset agent so we do not save cookies
         request = supertest.agent(test.cname);
         request
@@ -231,6 +239,14 @@ var moment = require("moment");
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));
             res.header["server"].should.match(new RegExp("postgrest", "g"));
+            should.exist(res.body.geojson);
+            /*
+            if (res.body.geojson.features == null) { // aava
+              //res.body.geojson.features.should.not.be.ok();
+              done(err);
+            }
+            res.body.geojson.features.length.should.be.equal(53);
+            */
             done(err);
           });
       });

tests/index6.js

@@ -183,6 +183,7 @@ var moment = require("moment");
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));
             res.header["server"].should.match(new RegExp("postgrest", "g"));
+            should.exist(res.body.geojson);
             done(err);
           });
       });
@@ -193,12 +194,13 @@ var moment = require("moment");
           .post(test.replay_full.url)
           .set("Accept", "application/json")
           .end(function (err, res) {
-            console.log(res.text);
+            console.log(res.body);
             res.status.should.equal(200);
             should.exist(res.header["content-type"]);
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));
             res.header["server"].should.match(new RegExp("postgrest", "g"));
+            should.exist(res.body.geojson);
             done(err);
           });
       });

tests/sql/anonymous.sql

@@ -18,3 +18,10 @@ SELECT api.ispublic_fn('kapla', 'public_logs', 1);
 SELECT api.ispublic_fn('kapla', 'public_logs', 3);
 SELECT api.ispublic_fn('kapla', 'public_monitoring');
 SELECT api.ispublic_fn('kapla', 'public_timelapse');
+
+SELECT api.ispublic_fn('aava', 'public_test');
+SELECT api.ispublic_fn('aava', 'public_logs_list');
+SELECT api.ispublic_fn('aava', 'public_logs', 1);
+SELECT api.ispublic_fn('aava', 'public_logs', 3);
+SELECT api.ispublic_fn('aava', 'public_monitoring');
+SELECT api.ispublic_fn('aava', 'public_timelapse');

tests/sql/anonymous.sql.output

@@ -21,6 +21,24 @@ ispublic_fn | f
 -[ RECORD 1 ]--
 ispublic_fn | t
 
+-[ RECORD 1 ]--
+ispublic_fn | t
+
+-[ RECORD 1 ]--
+ispublic_fn | f
+
+-[ RECORD 1 ]--
+ispublic_fn | f
+
+-[ RECORD 1 ]--
+ispublic_fn | f
+
+-[ RECORD 1 ]--
+ispublic_fn | t
+
+-[ RECORD 1 ]--
+ispublic_fn | t
+
 -[ RECORD 1 ]--
 ispublic_fn | f
 

tests/sql/cron_post_jobs.sql.output

@@ -17,7 +17,7 @@ count | 2
 
 logbook
 -[ RECORD 1 ]--+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-name           | Pojoviken to Norra hamnen
+name           | Pojoviken to Formanshagen
 _from_time     | t
 _to_time       | t
 track_geojson  | t
@@ -30,7 +30,7 @@ max_wind_speed | 22.1
 notes          | 
 extra          | {"metrics": {"propulsion.main.runTime": "PT10S"}, "observations": {"seaState": -1, "visibility": -1, "cloudCoverage": -1}, "avg_wind_speed": 14.549999999999999}
 -[ RECORD 2 ]--+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-name           | Norra hamnen to Ekenäs
+name           | Formanshagen to Ekenäs
 _from_time     | t
 _to_time       | t
 track_geojson  | t

tests/sql/grafana.sql.output

@@ -126,8 +126,8 @@ available_keys  |
 api.logs_view
 -[ RECORD 1 ]----+-----------------------
 id               | 2
-name             | Norra hamnen to Ekenäs
-from             | Norra hamnen
+name             | Formanshagen to Ekenäs
+from             | Formanshagen
 to               | Ekenäs
 distance         | 8.8968
 duration         | PT20M
@@ -137,7 +137,7 @@ _to_moorage_id   | 3
 id               | 1
 name             | patch log name 3
 from             | patch moorage name 3
-to               | Norra hamnen
+to               | Formanshagen
 distance         | 7.6447
 duration         | PT27M
 _from_moorage_id | 1
@@ -191,7 +191,7 @@ api.stays_view
 -[ RECORD 1 ]+---------------------
 id           | 2
 name         | t
-moorage      | Norra hamnen
+moorage      | Formanshagen
 moorage_id   | 2
 duration     | PT2M
 stayed_at    | Dock
@@ -226,7 +226,7 @@ notes     | new moorage note 3
 -[ RECORD 2 ]-------------------------------------------------
 id        | 2
 vessel_id | t
-name      | Norra hamnen
+name      | Formanshagen
 country   | fi
 stay_code | 4
 latitude  | 59.9768833333333
@@ -249,7 +249,7 @@ notes     |
 api.moorages_view
 -[ RECORD 1 ]-------+---------------------
 id                  | 2
-moorage             | Norra hamnen
+moorage             | Formanshagen
 default_stay        | Dock
 default_stay_id     | 4
 arrivals_departures | 2
@@ -287,7 +287,7 @@ stay_first_seen    | f
 stay_last_seen     | f
 -[ RECORD 2 ]------+---------------------------------------------------
 id                 | 2
-name               | Norra hamnen
+name               | Formanshagen
 default_stay       | Dock
 latitude           | 59.9768833333333
 longitude          | 23.4321

tests/sql/logbook.sql.output

@@ -30,7 +30,7 @@ max_wind_speed | 22.1
 notes          | new log note 3
 extra          | {"tags": ["tag_name"], "metrics": {"propulsion.main.runTime": "PT10S"}, "observations": {"seaState": -1, "visibility": -1, "cloudCoverage": 1}, "avg_wind_speed": 14.549999999999999}
 -[ RECORD 2 ]--+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-name           | Norra hamnen to Ekenäs
+name           | Formanshagen to Ekenäs
 _from_time     | t
 _to_time       | t
 track_geom     | 0102000020E610000013000000029A081B9E6E37404A5658830AFD4D404806A6C0EF6C3740DA1B7C6132FD4D40FE65F7E461693740226C787AA5FC4D407DD3E10EC1663740B29DEFA7C6FB4D40898BB63D5465374068479724BCFA4D409A5271F6E1633740B6847CD0B3F94D40431CEBE236623740E9263108ACF84D402C6519E2585F37407E678EBFC7F74D4096218E75715B374027C5B45C23F74D402AA913D044583740968DE1C46AF64D405AF5B9DA8A5537407BEF829B9FF54D407449C2ABD253374086C954C1A8F44D407D1A0AB278543740F2B0506B9AF34D409D11A5BDC15737406688635DDCF24D4061C3D32B655937402CAF6F3ADCF14D408988888888583740B3319C58CDF04D4021FAC8C0145837408C94405DB7EF4D40B8F9593F105B37403DC0804BEDEE4D40DE4C5FE2A25D3740AE47E17A14EE4D40

tests/sql/metadata.sql

@@ -29,4 +29,4 @@ UPDATE api.metadata SET configuration = '{ "depthKey": "environment.depth.belowT
 select configuration->'depthKey' AS depthKey, configuration->'update_at' IS NOT NULL AS update_at from api.metadata WHERE vessel_id = current_setting('vessel.id', false);
 
 \echo 'api.metadata get configuration base on update_at value'
-select configuration->'depthKey' AS depthKey, configuration->'update_at' IS NOT NULL AS update_at from api.metadata WHERE vessel_id = current_setting('vessel.id', false) AND configuration->>'update_at' = to_char(NOW(), 'YYYY-MM-DD"T"HH24:MI:SS"Z"');
+select configuration->'depthKey' AS depthKey, configuration->'update_at' IS NOT NULL AS update_at from api.metadata WHERE vessel_id = current_setting('vessel.id', false) AND configuration->>'update_at' <= to_char(NOW(), 'YYYY-MM-DD"T"HH24:MI:SS"Z"');

tests/sql/mobilitydb.sql.output

@@ -54,7 +54,7 @@ vessel_id | t
 Export timelapse as Geometry LineString from a trip
 -[ RECORD 1 ]--+-----------
 geometry_type  | LineString
-num_properties | 22
+num_properties | 26
 
 Export timelapse as Geometry Point from a trip
 -[ RECORD 1 ]

tests/sql/summary.sql.output

@@ -58,10 +58,15 @@ Schema      | public
 Description | PostGIS geometry and geography spatial types and functions
 -[ RECORD 10 ]-------------------------------------------------------------------------------------
 Name        | timescaledb
-Version     | 2.19.0
+Version     | 2.19.3
 Schema      | public
 Description | Enables scalable inserts and complex queries for time-series data (Community Edition)
 -[ RECORD 11 ]-------------------------------------------------------------------------------------
+Name        | timescaledb_toolkit
+Version     | 1.21.0
+Schema      | public
+Description | Library of analytical hyperfunctions, time-series pipelining, and other SQL utilities
+-[ RECORD 12 ]-------------------------------------------------------------------------------------
 Name        | uuid-ossp
 Version     | 1.1
 Schema      | public
@@ -671,16 +676,16 @@ overpass_py_fn | {"fee": "yes", "vhf": "09", "name": "Port Olímpic", "phone": "
 -[ RECORD 1 ]--+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 overpass_py_fn | {"name": "Port de la Ginesta", "type": "multipolygon", "leisure": "marina", "name:ca": "Port de la Ginesta", "wikidata": "Q16621038", "wikipedia": "ca:Port Ginesta", "check_date": "2024-08-23"}
 
--[ RECORD 1 ]--+----------------------------------------------
-overpass_py_fn | {"name": "Norra hamnen", "leisure": "marina"}
+-[ RECORD 1 ]--+---------------------------------------------------------------------------------------
+overpass_py_fn | {"leisure": "marina", "seamark:type": "harbour", "seamark:harbour:category": "marina"}
 
--[ RECORD 1 ]----------------------------------------------------------------------------------------------------------------------------------------------------------------------
-versions_fn | {"api_version" : "0.9.0", "sys_version" : "PostgreSQL 16.8", "mobilitydb" : "1.2.0", "timescaledb" : "2.19.0", "postgis" : "3.5.2", "postgrest" : "PostgREST 12.2.8"}
+-[ RECORD 1 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+versions_fn | {"api_version" : "0.9.1", "sys_version" : "PostgreSQL 16.8", "mobilitydb" : "1.2.0", "timescaledb" : "2.19.3", "postgis" : "3.5.2", "postgrest" : "PostgREST 12.2.12"}
 
--[ RECORD 1 ]-----------------
-api_version | 0.9.0
+-[ RECORD 1 ]------------------
+api_version | 0.9.1
 sys_version | PostgreSQL 16.8
-timescaledb | 2.19.0
+timescaledb | 2.19.3
 postgis     | 3.5.2
-postgrest   | PostgREST 12.2.8
+postgrest   | PostgREST 12.2.12