Release 0.7.6-202408

- Update public.process_lat_lon_fn, Ensure name without double quote, Fix issue with when string contains quote
- Update public.qgis_bbox_trip_py_fn, Add error handling for invalid argument
- Update public.qgis_bbox_py_fn, Add error handling for invalid argument
- Update public.get_app_settings_fn, Add new qgis and video URLs settings
- Update public.process_post_logbook_fn, Remove qgis dependency, Add logbook_stats
- Update public.cron_alerts_fn, Update alert message. Add the date and format the numeric value
- Update public.metrics_trigger_fn, CleanUp DEBUG print, ensure only one "new_stay" insert.
- Update public.check_jwt, reset config value Set default settings to avoid sql shared session cache for every new HTTP request
- Update public.send_email_py_fn, Update HTML email for new logbook email
- Update deactivated email template
- Update new logbook email template
- Update role authenticator, set api role SQL connection to 30
- Update table auth.vessels, Remove CONSTRAINT unique email for vessels table, Create new index for vessels table

.github/workflows/db-lint.yml

@@ -27,7 +27,7 @@ jobs:
         run: cp .env.example .env
 
       - name: Pull Docker images
-        run: docker-compose pull db api
+        run: docker compose pull db api
 
       - name: Run PostgSail Database & schemalint
         # Environment variables
@@ -41,10 +41,10 @@ jobs:
         run: |
           set -eu
           source .env
-          docker-compose stop || true
+          docker compose stop || true
-          docker-compose rm || true
+          docker compose rm || true
-          docker-compose up -d db && sleep 30 && docker-compose up -d api && sleep 5
+          docker compose up -d db && sleep 30 && docker compose up -d api && sleep 5
-          docker-compose ps -a
+          docker compose ps -a
           echo ${PGSAIL_API_URL}
           curl ${PGSAIL_API_URL}
           npm i -D schemalint
@@ -52,4 +52,4 @@ jobs:
       - name: Show the logs
         if: always()
         run: |
-          docker-compose logs
+          docker compose logs

.github/workflows/db-test.yml

@@ -29,10 +29,10 @@ jobs:
         run: cp .env.example .env
 
       - name: Pull Docker images
-        run: docker-compose pull db api
+        run: docker compose pull db api
 
       - name: Build Docker images
-        run: docker-compose -f docker-compose.dev.yml -f docker-compose.yml build tests
+        run: docker compose -f docker-compose.dev.yml -f docker-compose.yml build tests
 
       - name: Install psql
         run: sudo apt install postgresql-client
@@ -49,10 +49,10 @@ jobs:
         run: |
           set -eu
           source .env
-          docker-compose stop || true
+          docker compose stop || true
-          docker-compose rm || true
+          docker compose rm || true
-          docker-compose up -d db && sleep 30 && docker-compose up -d api && sleep 5
+          docker compose up -d db && sleep 30 && docker compose up -d api && sleep 5
-          docker-compose ps -a
+          docker compose ps -a
           echo ${PGSAIL_API_URL}
           curl ${PGSAIL_API_URL}
           psql -c "select 1"
@@ -70,4 +70,4 @@ jobs:
       - name: Show the logs
         if: always()
         run: |
-          docker-compose logs
+          docker compose logs

.github/workflows/frontend-test.yml

@@ -49,10 +49,10 @@ jobs:
         run: |
           set -eu
           source .env
-          docker-compose stop || true
+          docker compose stop || true
-          docker-compose rm || true
+          docker compose rm || true
-          docker-compose up -d db && sleep 30 && docker-compose up -d api && sleep 5
+          docker compose up -d db && sleep 30 && docker compose up -d api && sleep 5
-          docker-compose ps -a
+          docker compose ps -a
           echo "Test PostgSail Web Unit Test"
           docker compose -f docker-compose.dev.yml -f docker-compose.yml up -d web_dev && sleep 100
           docker compose -f docker-compose.dev.yml -f docker-compose.yml logs web_dev
@@ -67,4 +67,4 @@ jobs:
       - name: Show the logs
         if: always()
         run: |
-          docker-compose logs
+          docker compose logs

docker-compose.yml

@@ -47,6 +47,8 @@ services:
       PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000
       PGRST_DB_PRE_REQUEST: public.check_jwt
       PGRST_DB_POOL: 20
+      PGRST_DB_POOL_MAX_IDLETIME: 60
+      PGRST_DB_POOL_ACQUISITION_TIMEOUT: 20
       PGRST_DB_URI: ${PGRST_DB_URI}
       PGRST_JWT_SECRET: ${PGRST_JWT_SECRET}
       PGRST_SERVER_TIMING_ENABLED: 1

docs/ERD/README.md

@@ -124,5 +124,13 @@ SELECT * from public.process_queue;
 If you just want to use this as a standalone application and don't want people to be able to sign up for an account.
 
 ```SQL
-revoke execute on function api.signup(text,text,text,text) to api_anonymous;
+REVOKE execute on function api.signup(text,text,text,text) to api_anonymous;
+```
+
+### How to disable completely anonymous access
+
+If you just want to use this as a standalone application and don't want people to be able to access public account.
+
+```SQL
+REVOKE SELECT ON ALL TABLES IN SCHEMA api TO api_anonymous;
 ```

docs/ERD/postgsail.md

@@ -257,5 +257,5 @@ erDiagram
     api_stays }o--|| api_moorages : ""
     api_stays }o--|| api_stays_at : ""
     auth_otp |o--|| auth_accounts : ""
-    auth_vessels |o--|| auth_accounts : ""
+    auth_vessels }o--|| auth_accounts : ""
 ```

docs/README.md

@@ -190,7 +190,7 @@ Check the [End-to-End (E2E) test sample](https://github.com/xbgmsharp/postgsail/
 
 ### Docker dependencies
 
-`docker-compose` is used to start environment dependencies. Dependencies consist of 3 containers:
+`docker compose` is used to start environment dependencies. Dependencies consist of 3 containers:
 
 - `timescaledb-postgis` alias `db`, PostgreSQL with TimescaleDB extension along with the PostGIS extension.
 - `postgrest` alias `api`, Standalone web server that turns your PostgreSQL database directly into a RESTful API.

initdb/99_migrations_202407.sql

@@ -708,7 +708,7 @@ COMMENT ON FUNCTION
     public.cron_process_no_activity_fn
     IS 'init by pg_cron, check for vessel with no activity for more than 230 days then send notification';
 
--- Update grafana role SQL connection to 30
+-- Update grafana,qgis,api role SQL connection to 30
 ALTER ROLE grafana WITH NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS NOREPLICATION CONNECTION LIMIT 30 LOGIN;
 ALTER ROLE api_anonymous WITH NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS NOREPLICATION CONNECTION LIMIT 30 LOGIN;
 ALTER ROLE qgis_role WITH NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS NOREPLICATION CONNECTION LIMIT 30 LOGIN;

initdb/PGSAIL_VERSION

@@ -1 +1 @@
-0.7.5
+0.7.6

tests/README.md

@@ -13,6 +13,6 @@ $ bash tests.sh
 
 ## docker
 ```bash
-$ docker-compose up -d db && sleep 15 && docker-compose up -d api && sleep 5
+$ docker compose up -d db && sleep 15 && docker compose up -d api && sleep 5
-$ docker-compose -f docker-compose.dev.yml -f docker-compose.yml up tests
+$ docker compose -f docker-compose.dev.yml -f docker-compose.yml up tests
 ```

tests/index5.js

@@ -142,7 +142,7 @@ var moment = require("moment");
           .set(test.logs.header.name, test.logs.header.value)
           .set("Accept", "application/json")
           .end(function (err, res) {
-            res.status.should.equal(404);
+            res.status.should.equal(200);
             should.exist(res.header["content-type"]);
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));
@@ -192,7 +192,7 @@ var moment = require("moment");
           .set("Accept", "application/json")
           .end(function (err, res) {
             console.log(res.text);
-            res.status.should.equal(404); // return 404 as it is not enable in user settings.
+            res.status.should.equal(200); // return 404 as it is not enable in user settings.
             should.exist(res.header["content-type"]);
             should.exist(res.header["server"]);
             res.header["content-type"].should.match(new RegExp("json", "g"));

tests/index6.js

@@ -0,0 +1,203 @@
+"use strict";
+/*
+ * Unit test #5
+ * Public/Anonymous access
+ *
+ * process.env.PGSAIL_API_URI = from inside the docker
+ *
+ * npm install supertest should mocha mochawesome moment
+ * alias mocha="./node_modules/mocha/bin/_mocha"
+ * mocha index5.js --reporter mochawesome --reporter-options reportDir=/mnt/postgsail/,reportFilename=report_api.html
+ *
+ */
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+const supertest = require("supertest");
+// Deprecated
+const should = require("should");
+//const chai = require("chai");
+//const should = chai.should();
+let request = null;
+var moment = require("moment");
+
+// Users Array
+[
+  {
+    cname: process.env.PGSAIL_API_URI,
+    name: "PostgSail unit test anonymous, no x-is-public header",
+    moorages: {
+      url: "/moorages_view",
+      payload: null,
+      res: {},
+    },
+    stays: {
+      url: "/stays_view",
+      payload: null,
+      res: {},
+    },
+    logs: {
+      url: "/logs_view",
+      payload: null,
+      res: {},
+    },
+    log: {
+      url: "/log_view?id=eq.1",
+      payload: null,
+      res: {},
+    },
+    monitoring: {
+      url: "/monitoring_view",
+      payload: null,
+      res: {},
+    },
+    timelapse: {
+      url: "/rpc/timelapse_fn",
+      payload: null,
+      res: {},
+    },
+    timelapse_full: {
+      url: "/rpc/timelapse_fn",
+      payload: null,
+      res: {},
+    },
+    stats_logs: {
+      url: "/rpc/stats_logs_fn",
+      payload: null,
+      res: {},
+    },
+    stats_stays: {
+      url: "/rpc/stats_stay_fn",
+      payload: null,
+      res: {},
+    },
+    export_gpx: {
+      url: "/rpc/export_logbook_gpx_fn",
+      payload: null,
+      res: {},
+    },
+  },
+].forEach(function (test) {
+  //console.log(`${test.cname}`);
+  describe(`${test.name}`, function () {
+    request = supertest.agent(test.cname);
+    request.set("User-Agent", "PostgSail unit tests");
+
+    describe("With no JWT as api_anonymous, no x-is-public", function () {
+      it("/stays_view, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .get(test.stays.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            res.body.length.should.be.equal(0);
+            done(err);
+          });
+      });
+      it("/moorages_view, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .get(test.log.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            res.body.length.should.be.equal(0);
+            done(err);
+          });
+      });
+      it("/logs_view, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .get(test.logs.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            res.body.length.should.be.equal(0);
+            done(err);
+          });
+      });
+      it("/log_view, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .get(test.log.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            res.body.length.should.be.equal(0);
+            done(err);
+          });
+      });
+      it("/monitoring_view, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .get(test.monitoring.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            console.log(res.text);
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            res.body.length.should.be.equal(0);
+            done(err);
+          });
+      });
+      it("/rpc/timelapse_fn, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .post(test.timelapse.url)
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            console.log(res.text);
+            res.status.should.equal(200);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            done(err);
+          });
+      });
+      it("/rpc/export_logbook_gpx_fn, api_anonymous no jwt token", function (done) {
+        // Reset agent so we do not save cookies
+        request = supertest.agent(test.cname);
+        request
+          .post(test.export_gpx.url)
+          .send({_id: 1})
+          .set("Accept", "application/json")
+          .end(function (err, res) {
+            console.log(res.text)
+            res.status.should.equal(401);
+            should.exist(res.header["content-type"]);
+            should.exist(res.header["server"]);
+            res.header["content-type"].should.match(new RegExp("json", "g"));
+            res.header["server"].should.match(new RegExp("postgrest", "g"));
+            done(err);
+          });
+      });
+    }); // user JWT
+  }); // OpenAPI description
+}); // Users Array

tests/sql/cron_post_jobs.sql.output

@@ -66,7 +66,7 @@ stay_code | 4
 
 eventlogs_view
 -[ RECORD 1 ]
-count | 12
+count | 11
 
 stats_logs_fn
 SELECT 1

tests/sql/cron_run_jobs.sql

@@ -13,7 +13,7 @@ select current_database();
 
 -- Check the number of process pending
 \echo 'Check the number of process pending'
--- Should be 22
+-- Should be 24
 SELECT count(*) as jobs from public.process_queue pq where pq.processed is null;
 --set role scheduler
 SELECT public.run_cron_jobs();

tests/sql/cron_run_jobs.sql.output

@@ -7,7 +7,7 @@ You are now connected to database "signalk" as user "username".
 Expanded display is on.
 Check the number of process pending
 -[ RECORD 1 ]
-jobs | 26
+jobs | 24
 
 -[ RECORD 1 ]-+-
 run_cron_jobs | 

tests/sql/summary.sql.output

@@ -6,7 +6,7 @@
 You are now connected to database "signalk" as user "username".
 Expanded display is on.
 -[ RECORD 1 ]--+-------------------------------
-server_version | 16.3 (Debian 16.3-1.pgdg120+1)
+server_version | 16.4 (Debian 16.4-1.pgdg120+1)
 
 -[ RECORD 1 ]--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 postgis_full_version | POSTGIS="3.4.2 c19ce56" [EXTENSION] PGSQL="160" GEOS="3.11.1-CAPI-1.17.1" PROJ="9.1.1 NETWORK_ENABLED=OFF URL_ENDPOINT=https://cdn.proj.org USER_WRITABLE_DIRECTORY=/var/lib/postgresql/.local/share/proj DATABASE_PATH=/usr/share/proj/proj.db" LIBXML="2.9.14" LIBJSON="0.16" LIBPROTOBUF="1.4.1" WAGYU="0.5.0 (Internal)"
@@ -53,7 +53,7 @@ Schema      | public
 Description | PostGIS geometry and geography spatial types and functions
 -[ RECORD 9 ]--------------------------------------------------------------------------------------
 Name        | timescaledb
-Version     | 2.15.3
+Version     | 2.16.1
 Schema      | public
 Description | Enables scalable inserts and complex queries for time-series data (Community Edition)
 -[ RECORD 10 ]-------------------------------------------------------------------------------------
@@ -106,14 +106,14 @@ laninline     | 13566
 lanvalidator  | 13567
 lanacl        | 
 -[ RECORD 5 ]-+-----------
-oid           | 18168
+oid           | 18191
 lanname       | plpython3u
 lanowner      | 10
 lanispl       | t
 lanpltrusted  | t
-lanplcallfoid | 18165
+lanplcallfoid | 18188
-laninline     | 18166
+laninline     | 18189
-lanvalidator  | 18167
+lanvalidator  | 18190
 lanacl        | 
 
 -[ RECORD 1 ]+-----------
@@ -656,19 +656,19 @@ Test opverpass API overpass_py_fn
 -[ RECORD 1 ]--+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 overpass_py_fn | {"fee": "yes", "vhf": "09", "name": "Port Olímpic", "phone": "+34 933561016", "leisure": "marina", "website": "https://portolimpic.barcelona/", "wikidata": "Q171204", "wikipedia": "ca:Port Olímpic de Barcelona", "addr:street": "Moll de Xaloc", "power_supply": "yes", "seamark:type": "harbour", "addr:postcode": "08005", "internet_access": "wlan", "wikimedia_commons": "Category:Port Olímpic (Barcelona)", "sanitary_dump_station": "yes", "seamark:harbour:category": "marina"}
 
--[ RECORD 1 ]--+----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+-[ RECORD 1 ]--+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-overpass_py_fn | {"name": "Port de la Ginesta", "type": "multipolygon", "leisure": "marina", "name:ca": "Port de la Ginesta", "wikidata": "Q16621038", "wikipedia": "ca:Port Ginesta"}
+overpass_py_fn | {"name": "Port de la Ginesta", "type": "multipolygon", "leisure": "marina", "name:ca": "Port de la Ginesta", "wikidata": "Q16621038", "wikipedia": "ca:Port Ginesta", "check_date": "2024-08-23"}
 
 -[ RECORD 1 ]--+----------------------------------------------
 overpass_py_fn | {"name": "Norra hamnen", "leisure": "marina"}
 
 -[ RECORD 1 ]----------------------------------------------------------------------------------------------------------------------------------------------
-versions_fn | {"api_version" : "0.7.4", "sys_version" : "PostgreSQL 16.3", "timescaledb" : "2.15.3", "postgis" : "3.4.2", "postgrest" : "PostgREST 12.2.2"}
+versions_fn | {"api_version" : "0.7.6", "sys_version" : "PostgreSQL 16.4", "timescaledb" : "2.16.1", "postgis" : "3.4.2", "postgrest" : "PostgREST 12.2.3"}
 
 -[ RECORD 1 ]-----------------
-api_version | 0.7.4
+api_version | 0.7.6
-sys_version | PostgreSQL 16.3
+sys_version | PostgreSQL 16.4
-timescaledb | 2.15.3
+timescaledb | 2.16.1
 postgis     | 3.4.2
-postgrest   | PostgREST 12.2.2
+postgrest   | PostgREST 12.2.3
 

tests/tests.sh

@@ -168,6 +168,14 @@ else
     echo mocha index5.js
     exit 1
 fi
+# Anonymous API unit tests
+$mymocha index6.js --reporter ./node_modules/mochawesome --reporter-options reportDir=output/,reportFilename=report6.html
+if [ $? -eq 0 ]; then
+    echo OK
+else
+    echo mocha index6.js
+    exit 1
+fi
 
 # Anonymous SQL unit tests
 psql ${PGSAIL_DB_URI} < sql/anonymous.sql > output/anonymous.sql.output