From bd5cf9978db68fa4a344f8d45616c14275582e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhard=20X=2E=20F=C3=BCrst?= Date: Fri, 31 Jul 2026 07:25:54 +0000 Subject: [PATCH] readin/readarchive: gemeinsame Module, Stack-Konventionen mongo.js, influx_post.js und logit.js lagen in beiden Komponenten doppelt und waren auseinandergelaufen - so ist der Zeitzonen-Fehler entstanden, den readin seit jeher richtig loest und readFromcsv.js zwei Jahre lang nicht. Sie liegen jetzt einmal unter common/. Wo die Fassungen sich widersprachen: - MONGOAUTH wird als String verglichen. Die readarchive-Fassung pruefte nur auf truthy, dadurch schaltete auch MONGOAUTH=false die Auth ein. - writeDataArray(client, coll, data) nimmt den Collection-Namen direkt; readarchive baut ihn mit dem neuen dataCollName(styp). - getallProperties liefert das Array selbst. Die readarchive-Fassung mit {error, properties} war dort ungenutzt. - properties_collection heisst einheitlich property_coll. Das Sammelobjekt statistics lag in readin/readdata.js und wurde von mongo.js und influx_post.js importiert. Es liegt jetzt in common/statistics.js, damit die gemeinsamen Module readin nicht kennen muessen. Der Build-Kontext beider Images ist dadurch das Repository-Wurzelverzeichnis; das Layout im Image spiegelt das Repository, damit ../common/... unveraendert aufgeht. Die Volume-Zeile von readin im Compose zieht deshalb auf /opt/app/readin/data um. readarchive folgt jetzt den Konventionen des Stacks: deploy.sh statt build_and_copy.sh, Dockerfile_readarchive statt Dockerfile_rfcsv, und die eigene docker-compose.yml entfaellt - der Dienst haengt als Profil "tools" im Compose des Stacks und wird mit Parametern gestartet: docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29 Getestet: beide Images gebaut, beide gegen Testdatenbanken laufen lassen. Nicht deployt - die Images in der Registry sind unveraendert. Co-Authored-By: Claude Opus 5 (1M context) --- .dockerignore | 10 + {readarchive => common}/influx_post.js | 35 +--- {readarchive/utilities => common}/logit.js | 1 - {readin => common}/mongo.js | 174 +++++++++++++--- common/statistics.js | 7 + noisesensors/docker-compose.yml | 26 ++- ...ockerfile_rfcsv => Dockerfile_readarchive} | 24 ++- readarchive/build_and_copy.sh | 63 ------ readarchive/crontab.tmp | 2 - readarchive/deploy.sh | 63 ++++++ readarchive/docker-compose.yml | 56 ----- readarchive/logit.js | 12 -- readarchive/mongo.js | 197 ------------------ readarchive/mqtt.js | 2 +- readarchive/readFromcsv.js | 10 +- readarchive/utilities/checkprops.js | 4 +- readarchive/utilities/reporterror.js | 2 +- readin/.dockerignore | 10 - readin/Dockerfile_readin | 19 +- readin/crontab.tmp | 3 +- readin/deploy.sh | 8 +- readin/fetchnewdata.js | 9 +- readin/influx_post.js | 100 --------- readin/logit.js | 12 -- readin/parse.js | 6 +- readin/readdata.js | 4 +- 26 files changed, 304 insertions(+), 555 deletions(-) create mode 100644 .dockerignore rename {readarchive => common}/influx_post.js (81%) rename {readarchive/utilities => common}/logit.js (99%) rename {readin => common}/mongo.js (57%) create mode 100644 common/statistics.js rename readarchive/{Dockerfile_rfcsv => Dockerfile_readarchive} (54%) delete mode 100755 readarchive/build_and_copy.sh delete mode 100644 readarchive/crontab.tmp create mode 100755 readarchive/deploy.sh delete mode 100644 readarchive/docker-compose.yml delete mode 100644 readarchive/logit.js delete mode 100644 readarchive/mongo.js delete mode 100644 readin/.dockerignore delete mode 100644 readin/influx_post.js delete mode 100644 readin/logit.js diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..40492b9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +**/node_modules +**/data +**/log +**/*.log +**/.env +noise +noisesensors +sensorapi +noise.tgz diff --git a/readarchive/influx_post.js b/common/influx_post.js similarity index 81% rename from readarchive/influx_post.js rename to common/influx_post.js index 555d137..8416ef2 100644 --- a/readarchive/influx_post.js +++ b/common/influx_post.js @@ -1,7 +1,11 @@ -// Access to influxDB vie HTTP +/* Zugriff auf InfluxDB per HTTP + +Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv). +*/ import axios from 'axios' import { logit, logerror } from './logit.js' +import { statistics } from './statistics.js' import { DateTime } from 'luxon' let INFLUXHOST = process.env.INFLUXHOST || "localhost" @@ -14,7 +18,7 @@ const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${IN const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms` // `${e}` liefert bei einem AggregateError nur "AggregateError" - die einzelnen -// Verbindungsfehler (je einer pro aufgelöster IP) stecken in e.errors bzw. e.cause.errors +// Verbindungsfehler (je einer pro aufgeloester IP) stecken in e.errors bzw. e.cause.errors function describeError(e) { const sub = e.errors || e.cause?.errors if (sub?.length) { @@ -52,7 +56,6 @@ export const influxRead = async (query) => { return data } - // liefert true, wenn Influx die Daten uebernommen hat, sonst false export const influxWrite = async (data) => { let start = DateTime.now() @@ -77,31 +80,7 @@ export const influxWrite = async (data) => { } catch (e) { logerror(`doWrite2API ${INFLUXURL_WRITE} ${describeError(e)}`) } + statistics['writeInfluxData[sensor_data]Time'] = DateTime.now().diff(start, ['seconds']).toObject().seconds logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`) return ok } - -/* -async function main() { - let data = ` - pm,sid=140 P1=12,P2=13 - pm,sid=142 P1=42,P2=13 - pm,sid=143 P1=43,P2=13 - pm,sid=144 P1=44,P2=13 - thp,sid=141 temperature=23.5,humidity=48,pressure=998 - ` - let ret = await influxWrite(data) - process.exit() - - let query = `from(bucket:"sensor_data") -|> range(start: -1mo) -|> filter(fn: (r) => r._measurement == "pm") -|> filter(fn: (r) => r.sid == "140") -` - let erg = await influxRead(query) - console.log(erg) -} - - -main().catch(console.error) -*/ diff --git a/readarchive/utilities/logit.js b/common/logit.js similarity index 99% rename from readarchive/utilities/logit.js rename to common/logit.js index 7a870e4..0f2c946 100644 --- a/readarchive/utilities/logit.js +++ b/common/logit.js @@ -13,4 +13,3 @@ export function logerror(str) { let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`; console.log(s); } - diff --git a/readin/mongo.js b/common/mongo.js similarity index 57% rename from readin/mongo.js rename to common/mongo.js index f90fa27..e979fbd 100644 --- a/readin/mongo.js +++ b/common/mongo.js @@ -1,8 +1,19 @@ /* Interface for MongoDB + +Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv). +Zusammengefuehrt aus den beiden zuvor getrennten Fassungen; wo sie sich +unterschieden, gilt: + - MONGOAUTH wird als String verglichen. Die alte readarchive-Fassung + pruefte nur auf truthy, dadurch schaltete auch MONGOAUTH=false die + Authentifizierung ein. + - writeDataArray(client, coll, data) nimmt den Collection-Namen direkt. + readarchive baut ihn mit dataCollName(styp). + - getallProperties liefert das Array selbst (readin-Fassung). Die + readarchive-Fassung mit {error, properties} war dort ungenutzt. */ import { MongoClient } from 'mongodb' import { logit, logerror } from './logit.js' -import { statistics } from './readdata.js' +import { statistics } from './statistics.js' import { DateTime } from 'luxon' let DEVELOP = process.env.DEVELOP || 'false' @@ -13,13 +24,18 @@ let MONGOAUTH = process.env.MONGOAUTH || 'false' let MONGOUSRP = process.env.MONGOUSRP || '' let MONGOBASE = process.env.MONGOBASE || 'sensor_data' - let MONGO_URL = 'mongodb://' + MONGOHOST + ':' + MONGOPORT; // URL to mongo database if (MONGOAUTH == 'true') { MONGO_URL = 'mongodb://' + MONGOUSRP + '@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin'; // URL to mongo database } export const property_coll = 'properties' +const data_collection = 'sensors' + +// Datencollection eines Sensortyps, z.B. dataCollName('noise') -> 'noise_sensors' +export const dataCollName = (styp) => { + return styp + '_' + data_collection +} const addandshowstatistics = (client, text, field, start) => { statistics[field] = DateTime.now().diff(start, ['seconds']).toObject().seconds @@ -33,7 +49,7 @@ export const connectMongo = async () => { } else { logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`) } - let client = await MongoClient.connect(MONGO_URL, { useNewUrlParser: true, useUnifiedTopology: true }) + let client = await MongoClient.connect(MONGO_URL) if ( DEVELOP === 'true') { logit(`Mongodbase connected to ${MONGO_URL}`) } else { @@ -47,24 +63,107 @@ export const connectMongo = async () => { } export const closeMongo = async (client) => { - client.close() + try { + await client.close() + } + catch(error){ + throw(error) + } } +/* *************************************************** +// READ routines +******************************************************/ + +// Read properties from the database - oeffnet und schliesst selbst +export const readProperties = async (query, limit = 0) => { + let ret = {err: null, properties: null} + let client = await connectMongo() + try { + if ("sid" in query) { // if sid is given, read property for sid + ret.properties = await client.db(MONGOBASE).collection(property_coll).findOne({_id: query.sid}) + } else { // otherwise read props corresponding to query + ret.properties = await client.db(MONGOBASE).collection(property_coll).find(query).limit(limit).toArray() + } + } catch (e) { + ret.err = e + } + finally { + client.close() + } + return ret +} export const getallProperties = async (client) => { return await client.db(MONGOBASE).collection(property_coll) .find().sort({ _id: 1 }).toArray() - } export const checkOneproperty = async (client, sid) => { - return await client.db(MONGOBASE).collection("properties") + return await client.db(MONGOBASE).collection(property_coll) .findOne({ _id: sid }) } +export const getOneproperty = async (client, sid) => { + let ret = {error: false, errortext: '', property: null} + try { + ret.property = await client.db(MONGOBASE).collection(property_coll) + .findOne({_id: sid}) + } catch (e) { + ret = {error: true, errortext: e} + } + return ret +} + +export const getOneSensorOneday = async (client, sid, day, styp) => { + let ret = {error: false, errortext: '', date: []} + // muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC), + // sonst passt das Suchfenster nicht auf die Daten des Tages + let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' }) + let start = d.startOf('day').toJSDate() + let end = d.startOf('day').plus({day:1}).toJSDate() + try { + let erg = await client.db(MONGOBASE).collection(dataCollName(styp)) + .find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray() + ret.data = erg + } catch(e) { + ret.error = true + ret.errortext = e + } + return ret +} + +export const getLocationIDs = async (client, stype) => { + let ret = {error: false, errortext: '', locations: []} + try { + ret.locations = await client.db(MONGOBASE).collection(property_coll) + .distinct('location.0.id', {type: stype}) + } catch (e) { + ret.error = true + ret.errortext = e + } + return ret +} + +export const getTHPSensors = async (client, lid) => { + let ret = {error: false, errortext: '', sensors: []} + try { + ret.sensors = await client.db(MONGOBASE).collection(property_coll) + .findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}}) + } catch (e) { + ret.error = true + ret.errortext = e + } + return ret +} + +/* *************************************************** +// WRITE routines +******************************************************/ + export const writeOneproperty = async (client, prop) => { try { - let result = await client.db(MONGOBASE).collection("properties") + let result = await client.db(MONGOBASE).collection(property_coll) .insertOne(prop) } catch (e) { if (e.code == 11000) { @@ -80,7 +179,7 @@ export const writeProperties = async (client, props) => { let result let startAll = DateTime.now(); let start - let coll = client.db(MONGOBASE).collection("properties") + let coll = client.db(MONGOBASE).collection(property_coll) if (props.new.length !== 0) { start = DateTime.now(); try { @@ -120,6 +219,19 @@ export const writeProperties = async (client, props) => { addandshowstatistics(client, 'properties', 'writePropsTime', startAll) } +export const writeOneSensor = async (client, data, styp) => { + let ret = {error: false, errortext: '', inserted: 1} + try { + await client.db(MONGOBASE).collection(dataCollName(styp)) + .insertOne(data) + } catch (e) { + ret.error = true + ret.errortext = e + ret.inserted = 0 + } + return ret +} + export const writeDataArray = async (client, coll, data) => { let result let start = DateTime.now(); @@ -131,11 +243,7 @@ export const writeDataArray = async (client, coll, data) => { console.error(e) } } - let statname = `writeMongoData[${coll}]Time` addandshowstatistics(client, `${data.length} entries for ${coll}`, `writeMongoData[${coll}]Time`, start) - -// statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds -// logit(`Write Data for ${coll} to mongoDB: Time: ${statistics[statname]} sec.`) } export const writeStatistic = async (client, stat) => { @@ -151,27 +259,17 @@ export const writeStatistic = async (client, stat) => { addandshowstatistics(client, `statistics`, `writeStatisticTime`, start) } -export const dropColl = async (client, coll) => { +export const bulkWrite = async (client, coll, data) => { let start = DateTime.now() let result try { - result = await client.db(MONGOBASE).collection(coll).drop() + result = await client.db(MONGOBASE).collection(coll) + .bulkWrite(data, { ordered: false }) } catch (e) { console.error(e) } - logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`) -} - - -export const createIndex = async (client, coll) => { - let result - let start = DateTime.now() - try { - result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" }) - } catch (e) { - console.error(e) - } - logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`) + addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start) + return result } export const bulkUpdateMapdata = async (client, data) => { @@ -187,16 +285,28 @@ export const bulkUpdateMapdata = async (client, data) => { return result } +/* *************************************************** +// Wartung +******************************************************/ -export const bulkWrite = async (client, coll, data) => { +export const dropColl = async (client, coll) => { let start = DateTime.now() let result try { - result = await client.db(MONGOBASE).collection(coll) - .bulkWrite(data, { ordered: false }) + result = await client.db(MONGOBASE).collection(coll).drop() } catch (e) { console.error(e) } - addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start) - return result + logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`) +} + +export const createIndex = async (client, coll) => { + let result + let start = DateTime.now() + try { + result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" }) + } catch (e) { + console.error(e) + } + logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`) } diff --git a/common/statistics.js b/common/statistics.js new file mode 100644 index 0000000..c05a7ab --- /dev/null +++ b/common/statistics.js @@ -0,0 +1,7 @@ +/* Gemeinsames Sammelobjekt fuer Laufzeit-Statistiken. + +Lag frueher in readin/readdata.js. Liegt hier, damit auch die gemeinsam +genutzten Module darauf schreiben koennen, ohne readin kennen zu muessen. +*/ + +export const statistics = {} diff --git a/noisesensors/docker-compose.yml b/noisesensors/docker-compose.yml index bb9b4e6..4714b01 100644 --- a/noisesensors/docker-compose.yml +++ b/noisesensors/docker-compose.yml @@ -43,10 +43,30 @@ services: INFLUXTOKEN: ${DOCKER_INFLUXTOKEN} volumes: - ${LOCALDIR}/noisesensors/log:/var/log - - ${LOCALDIR}/noisesensors/data:/opt/app/data - container_name: readin + # Pfad im Image ist jetzt /opt/app/readin, weil readin und readarchive + # sich common/ teilen und das Layout das Repository spiegelt + - ${LOCALDIR}/noisesensors/data:/opt/app/readin/data + container_name: readin restart: unless-stopped - + + # Kein Dauerdienst, sondern ein Lauf mit Parametern - liest rueckwirkend die + # Tagesdateien von archive.sensor.community ein: + # docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29 + # Das Profil verhindert, dass "docker compose up" ihn mitstartet. + readarchive: + image: docker.citysensor.de/readarchive + profiles: ["tools"] + environment: + MONGOHOST: mongodb + INFLUXHOST: influxdb + MONGOAUTH: "true" + MONGOUSRP: ${MONGO_ROOT_USERNAME}:${MONGO_ROOT_PASSWORD} + INFLUXTOKEN: ${DOCKER_INFLUXTOKEN} + volumes: + - ${LOCALDIR}/noisesensors/log:/var/log + - ${LOCALDIR}/noisesensors/data/readarchive:/opt/app/readarchive/data + restart: "no" + logrotate: image: linkyard/logrotate volumes: diff --git a/readarchive/Dockerfile_rfcsv b/readarchive/Dockerfile_readarchive similarity index 54% rename from readarchive/Dockerfile_rfcsv rename to readarchive/Dockerfile_readarchive index ee1be95..8c0aa75 100644 --- a/readarchive/Dockerfile_rfcsv +++ b/readarchive/Dockerfile_readarchive @@ -1,15 +1,21 @@ FROM node:22-alpine -ADD package.json package-lock.json /tmp/ +# Build-Kontext ist das Repository-Wurzelverzeichnis (siehe deploy.sh), weil +# readarchive und readin sich die Module unter common/ teilen. Das Layout im +# Image spiegelt das Repository, damit die relativen Importe (../common/...) +# unveraendert funktionieren. + +ADD readarchive/package.json readarchive/package-lock.json /tmp/ RUN cd /tmp && npm ci --omit=dev RUN mkdir -p /opt/app && cp -a /tmp/node_modules /tmp/package.json /opt/app/ -WORKDIR /opt/app -ADD *.js /opt/app/ -RUN mkdir /opt/app/utilities -ADD utilities/*.js /opt/app/utilities + +ADD common/*.js /opt/app/common/ +ADD readarchive/*.js /opt/app/readarchive/ +ADD readarchive/package.json /opt/app/readarchive/ +ADD readarchive/utilities/*.js /opt/app/readarchive/utilities/ # readFromcsv.js legt hier die Sensorliste des Tages ab (data/list.json) -RUN mkdir -p /opt/app/data +RUN mkdir -p /opt/app/readarchive/data # betrifft nur noch die Uhrzeit in den Log-Zeilen - die Messwerte selbst # werden seit dem UTC-Fix unabhaengig von der Zeitzone der Maschine gespeichert @@ -17,11 +23,9 @@ RUN apk add --no-cache tzdata ENV TZ=Europe/Berlin RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +WORKDIR /opt/app/readarchive + # Exec-Form: nur so werden Argumente aus "docker run ..." an das # Programm durchgereicht. CMD ist der Default, den eigene Argumente ersetzen. -# -# docker run --rm --env-file .env -e INFLUXHOST=influxdb -e MONGOHOST=mongo \ -# --network laerm_readfromcsv_to_database_default rfcsv -t noise -s 2026-07-25 -# ENTRYPOINT ["node", "./readFromcsv.js"] CMD ["-t", "noise"] diff --git a/readarchive/build_and_copy.sh b/readarchive/build_and_copy.sh deleted file mode 100755 index bb34212..0000000 --- a/readarchive/build_and_copy.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Build Docker-Container -# -# Call: buildit.sh name [target] -# -# The Dockerfile must be named like Dockerfile_name -# -# 2018-09-20 rxf -# - before sending docker image to remote, tag actual remote image -# -# 2018-09-14 rxf -# - first Version -# - -set -x -port="" -orgName=rfcsv -name=rfcsv - -usage() -{ - echo "Usage build_and_copy.sh [-p port] [-n name] target" - echo " Build docker container $name and copy to target" - echo "Params:" - echo " target: Where to copy the container to " - echo " -p port: ssh port (default 22)" - echo " -n name: new name for container (default: $orgName)" -} - -while getopts n:p:h? o -do - case "$o" in - n) name="$OPTARG";; - p) port="-p $OPTARG";; - h) usage; exit 0;; - *) usage; exit 1;; - esac -done -shift $((OPTIND-1)) - -while [ $# -gt 0 ]; do - if [[ -z "$target" ]]; then - target=$1 - shift - else - echo "bad option $1" - # exit 1 - shift - fi -done - -docker build -f Dockerfile_$orgName -t $name . - -dat=`date +%Y%m%d%H%M` - -if [ "$target" == "localhost" ] -then - docker tag $name $name:V_$dat - exit -fi - -ssh $port $target "docker tag $name $name:V_$dat" -docker save $name | bzip2 | pv | ssh $port $target 'bunzip2 | docker load' diff --git a/readarchive/crontab.tmp b/readarchive/crontab.tmp deleted file mode 100644 index 9aa5e15..0000000 --- a/readarchive/crontab.tmp +++ /dev/null @@ -1,2 +0,0 @@ -#33 10 * * * cd /opt/app && node ./readFromcsv.js -t laerm >>/var/log/readFrom.log 2>&1 -#53 10 * * * cd /opt/app && node ./readFromcsv.js -t radia >>/var/log/readFrom.log 2>&1 diff --git a/readarchive/deploy.sh b/readarchive/deploy.sh new file mode 100755 index 0000000..5ad78ef --- /dev/null +++ b/readarchive/deploy.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -e + +# Angepasst aus readin/deploy.sh +# Unterschied: Build-Kontext ist das Repository-Wurzelverzeichnis, weil +# readarchive und readin sich die Module unter common/ teilen. + +REGISTRY="docker.citysensor.de" +IMAGE_NAME="readarchive" +TAG="${1:-latest}" +PLATFORM="linux/amd64" +DOCKERFILE="readarchive/Dockerfile_readarchive" +FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}" +BUILD_DATE=$(date +%d.%m.%Y) + +# ins Repository-Wurzelverzeichnis wechseln (eine Ebene ueber diesem Skript) +cd "$(dirname "$0")/.." + +echo "==========================================" +echo "readarchive Deploy Script (amd64)" +echo "==========================================" +echo "Registry: ${REGISTRY}" +echo "Image: ${IMAGE_NAME}" +echo "Tag: ${TAG}" +echo "Platform: ${PLATFORM}" +echo "Dockerfile: ${DOCKERFILE}" +echo "Kontext: $(pwd)" +echo "Build-Datum: ${BUILD_DATE}" +echo "==========================================" +echo "" + +# Hinweis: Login wird uebersprungen, weil die Credentials fuer ${REGISTRY} +# bereits in ~/.docker/config.json liegen (Container laufen bereits aus dieser Registry). +# Bei Bedarf manuell: docker login ${REGISTRY} + +echo ">>> Baue Image (nur ${PLATFORM}, nativ) ..." +docker build \ + --platform "${PLATFORM}" \ + -f "${DOCKERFILE}" \ + -t "${FULL_IMAGE}" \ + . +echo "" + +echo ">>> Pushe ${FULL_IMAGE} ..." +docker push "${FULL_IMAGE}" +echo "" + +# Falls ein anderer Tag als 'latest' gebaut wurde, zusaetzlich latest setzen+pushen +if [ "${TAG}" != "latest" ]; then + echo ">>> Tagge zusaetzlich als :latest ..." + docker tag "${FULL_IMAGE}" "${REGISTRY}/${IMAGE_NAME}:latest" + docker push "${REGISTRY}/${IMAGE_NAME}:latest" + echo "" +fi + +echo "==========================================" +echo "Build + Push fertig: ${FULL_IMAGE}" +echo "==========================================" +echo "" +echo "readarchive ist kein Dauerdienst, sondern ein Lauf mit Parametern:" +echo " cd /opt/stacks/esprimo/noisesensors" +echo " docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29" diff --git a/readarchive/docker-compose.yml b/readarchive/docker-compose.yml deleted file mode 100644 index 46f2f3a..0000000 --- a/readarchive/docker-compose.yml +++ /dev/null @@ -1,56 +0,0 @@ -version: "3.9" -volumes: - mongo_vol: - influx_vol: - -services: - # rfcsv: - # image: rfcsv - # volumes: - # - /var/log/rfcsv:/var/log - # - /home/rxf/CERTS:certs - # environment: - # - MONGOHOST=207.180.224.98 - # - MONGOPORT=20019 - # #- MONGOAUTH=true - # #- "MONGOUSRP=rxf:5C5dB|m" - # - MONGOBASE=allsensors - # - TZ=Europe/Berlin - # container_name: rfcsv - # restart: always - - mongodb: - image: mongo - volumes: - - mongo_vol:/data/db - ports: - - "27017:27017" - container_name: mongo - #environment: - # - MONGO_INITDB_ROOT_USERNAME=${MONGO_ROOT_USER} - # - MONGO_INITDB_ROOT_PASSWORD=${MONGO_ROOT_PASSWD} - #command: '--auth' - restart: unless-stopped - - influx: - image: influxdb:2.7 - ports: - - '8086:8086' - volumes: - # /etc/influxdb2 NICHT mounten: das Entrypoint legt dort beim Setup das - # CLI-Profil "default" an. Ein vorhandenes Profil laesst den Setup - # scheitern - und danach loescht das Entrypoint bolt+engine im Volume. - - influx_vol:/var/lib/influxdb2 - environment: - # Werte kommen aus .env, damit App und Container dieselben benutzen - - DOCKER_INFLUXDB_INIT_MODE=setup - - DOCKER_INFLUXDB_INIT_USERNAME=${DOCKER_INFLUXDB_INIT_USERNAME} - - DOCKER_INFLUXDB_INIT_PASSWORD=${DOCKER_INFLUXDB_INIT_PASSWORD} - - DOCKER_INFLUXDB_INIT_ORG=${DOCKER_INFLUXDB_INIT_ORG} - - DOCKER_INFLUXDB_INIT_BUCKET=${DOCKER_INFLUXDB_INIT_BUCKET} - - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${INFLUXTOKEN} - restart: - unless-stopped - container_name: influxdb - - diff --git a/readarchive/logit.js b/readarchive/logit.js deleted file mode 100644 index 19d8b28..0000000 --- a/readarchive/logit.js +++ /dev/null @@ -1,12 +0,0 @@ -import { DateTime} from 'luxon' - -export function logit(str) { - let s = `${DateTime.now().toISO()} => ${str}`; - console.log(s); -} - -export function logerror(str) { - let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`; - console.log(s); -} - diff --git a/readarchive/mongo.js b/readarchive/mongo.js deleted file mode 100644 index 311f435..0000000 --- a/readarchive/mongo.js +++ /dev/null @@ -1,197 +0,0 @@ -/* Interface for MongoDB -*/ -import { MongoClient } from 'mongodb' -import { logit, logerror } from './utilities/logit.js' -import { DateTime } from 'luxon' -import {returnOnError} from "./utilities/reporterror.js"; - -const DEVELOP = process.env.DEVELOP || false; - -const MONGOHOST = process.env.MONGOHOST || 'localhost'; -const MONGOPORT = process.env.MONGOPORT || 27017; -const MONGOAUTH = process.env.MONGOAUTH || false; -const MONGOUSRP = process.env.MONGOUSRP || ""; -const MONGOBASE = process.env.MONGOBASE || 'sensor_data'; - -const MONGO_URL = MONGOAUTH ? 'mongodb://'+MONGOUSRP+'@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin' : 'mongodb://'+MONGOHOST+':'+MONGOPORT; // URL to mongo database -if (DEVELOP) { - console.log(`MongoURL = "${MONGO_URL}" and Database = ${MONGOBASE}`); -} -const statistics = {} - -export const properties_collection = 'properties' -const data_collection = 'sensors' - -export const connectMongo = async () => { - try { - if(DEVELOP === 'true') { - logit(`Try to connect to ${MONGO_URL}`) - } else { - logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`) - } - let client = await MongoClient.connect(MONGO_URL) - if ( DEVELOP === 'true') { - logit(`Mongodbase connected to ${MONGO_URL}`) - } else { - logit('Mongodbase connected') - } - return client - } - catch (error) { - throw (error) - } -} - -export const closeMongo = async (client) => { - try { - await client.close() - } - catch(error){ - throw(error) - } -} - -const listDatabases = async (client) => { - let databasesList = await client.db().admin().listDatabases(); - - console.log("Databases:"); - databasesList.databases.forEach(db => console.log(` - ${db.name}`)); -} - -/* *************************************************** -// READ routines -******************************************************/ - -// Read properties from the database -export const readProperties = async (query, limit = 0) => { - let ret = {err: null, properties: null} - let client = await connectMongo() - try { - if ("sid" in query) { // if sid is given, read property for sid - ret.properties = await client.db(MONGOBASE).collection(properties_collection).findOne({_id: query.sid}) - } else { // otherwise read props corresponding to query - ret.properties = await client.db(MONGOBASE).collection(properties_collection).find(query).limit(limit).toArray() - } - } catch (e) { - ret.err = e - } - finally { - client.close() - } - return ret -} - - -export const getallProperties = async (client) => { - let ret = {error: false, errortext: '', properties: []} - try - { - ret.properties = await client.db(MONGOBASE).collection(properties_collection) - .find().sort({_id: 1}).toArray() - } - catch(e) { - ret = {error: true, errortext: e} - } - return ret -} - - -export const getOneproperty = async (client, sid) => { - let ret = {error: false, errortext: '', property: null} - try { - ret.property = await client.db(MONGOBASE).collection(properties_collection) - .findOne({_id: sid}) - } catch (e) { - ret = {error: true, errortext: e} - } - return ret -} - -const getCollName = (coll, st) => { - return st + '_' + coll -} - -export const getOneSensorOneday = async (client, sid, day, styp) => { - let ret = {error: false, errortext: '', date: []} - // muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC), - // sonst passt das Suchfenster nicht auf die Daten des Tages - let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' }) - let start = d.startOf('day').toJSDate() - let end = d.startOf('day').plus({day:1}).toJSDate() - try { - let erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp)) - .find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray() - ret.data = erg - } catch(e) { - ret.error = true - ret.errortext = e - } - return ret -} - -export const writeOneSensor = async (client, data, styp) => { - let ret = {error: false, errortext: '', inserted: 1} - let erg - try { - erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp)) - .insertOne(data) - } catch (e) { - ret.error = true - ret.errortext = e - ret.inserted = 0 - } -return ret -} - -export const writeDataArray = async (client, data, styp) => { - let result - let start = DateTime.now(); - try { - result = await client.db(MONGOBASE).collection(getCollName(data_collection, styp)) - .insertMany(data, {ordered: false}) - } catch (e) { - if(e.code !== 11000) { - console.error(e) - } - } - let statname = `writeData${getCollName(data_collection, styp)}Time` - statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds - logit(`Write Data for ${getCollName(data_collection, styp)} to mongoDB: Time: ${statistics[statname]} sec.`) -} - -export const bulkWrite = async (client, coll, data) => { - let start = DateTime.now() - let result - try { - result = await client.db(MONGOBASE).collection(coll) - .bulkWrite(data, {ordered: false}) - } catch (e) { - console.error(e) - } - logit(`Write Properties: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`) - return result -} - -export const getLocationIDs = async (client, stype) => { - let ret = {error: false, errortext: '', locations: []} - try { - ret.locations = await client.db(MONGOBASE).collection(properties_collection) - .distinct('location.0.id', {type: stype}) - } catch (e) { - ret.error = true - ret.errortext = e - } - return ret -} - -export const getTHPSensors = async (client, lid) => { - let ret = {error: false, errortext: '', sensors: []} - try { - ret.sensors = await client.db(MONGOBASE).collection(properties_collection) - .findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}}) - } catch (e) { - ret.error = true - ret.errortext = e - } - return ret -} \ No newline at end of file diff --git a/readarchive/mqtt.js b/readarchive/mqtt.js index 0624e1a..da58013 100644 --- a/readarchive/mqtt.js +++ b/readarchive/mqtt.js @@ -1,7 +1,7 @@ // MQTT interface import * as mqtt from 'mqtt' -import { logit, logerror } from './logit.js' +import { logit, logerror } from '../common/logit.js' const MQTTHOST = process.env.MQTTHOST || 'rexfue.de' const MQTTPORT = process.env.MQTTPORT || 1883 diff --git a/readarchive/readFromcsv.js b/readarchive/readFromcsv.js index bb8ae1c..cd82af3 100644 --- a/readarchive/readFromcsv.js +++ b/readarchive/readFromcsv.js @@ -37,13 +37,13 @@ const MAXTRIES = 10 import 'dotenv/config' import axios from 'axios' import https from 'https' -import * as mongo from './mongo.js' -import * as influx from './influx_post.js' +import * as mongo from '../common/mongo.js' +import * as influx from '../common/influx_post.js' import fs from 'fs' import csv from 'csvtojson' import pkg from './package.json' with { type: "json" } import mod_getopt from 'posix-getopt' -import { logit, logerror } from './utilities/logit.js' +import { logit, logerror } from '../common/logit.js' import { DateTime, Duration } from 'luxon' import { checkProperties} from "./utilities/checkprops.js"; import { version } from "os" @@ -376,7 +376,7 @@ async function enterOneSensorinDB(client, sid, erg, dataline, day, styp, args) { } if((args.database === 'both') || (args.database === 'mongo')) { if(erg.length !== 0) { - await mongo.writeDataArray(client, erg, styp) + await mongo.writeDataArray(client, mongo.dataCollName(styp), erg) } } return erg.length @@ -514,7 +514,7 @@ async function main() { logit(`Inserted: ${inserted}`) // enter changed properties into database if(newProps.length > 0) { - await mongo.bulkWrite(client, mongo.properties_collection, newProps) + await mongo.bulkWrite(client, mongo.property_coll, newProps) } await mongo.closeMongo(client) if (failedInfluxWrites.length > 0) { diff --git a/readarchive/utilities/checkprops.js b/readarchive/utilities/checkprops.js index 4387025..a6759a6 100644 --- a/readarchive/utilities/checkprops.js +++ b/readarchive/utilities/checkprops.js @@ -1,7 +1,7 @@ import { DateTime } from 'luxon' -import { logit, logerror } from './logit.js' +import { logit, logerror } from '../../common/logit.js' import { returnOnError } from './reporterror.js' -import * as mongo from '../mongo.js' +import * as mongo from '../../common/mongo.js' // Check lat/lon and convert to float function checkLatLon(w) { diff --git a/readarchive/utilities/reporterror.js b/readarchive/utilities/reporterror.js index 2b3de74..eac3e59 100644 --- a/readarchive/utilities/reporterror.js +++ b/readarchive/utilities/reporterror.js @@ -1,4 +1,4 @@ -import {logit} from "./logit.js"; +import {logit} from "../../common/logit.js"; export const reportError = (message, errortext) => { message.error = true diff --git a/readin/.dockerignore b/readin/.dockerignore deleted file mode 100644 index 4786954..0000000 --- a/readin/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -node-modules -.gitignore -.dockerignore -build_and_copy.sh -docker-compose.yml -Dockerfile* -INFLUXSERVER -TRANSFERE -data - diff --git a/readin/Dockerfile_readin b/readin/Dockerfile_readin index 925c5b0..2622d6a 100644 --- a/readin/Dockerfile_readin +++ b/readin/Dockerfile_readin @@ -1,14 +1,20 @@ FROM node:22-alpine -ADD package.json /tmp/package.json -ADD package-lock.json /tmp/package-lock.json +# Build-Kontext ist das Repository-Wurzelverzeichnis (siehe deploy.sh), weil +# readin und readarchive sich die Module unter common/ teilen. Das Layout im +# Image spiegelt das Repository, damit die relativen Importe (../common/...) +# unveraendert funktionieren. + +ADD readin/package.json /tmp/package.json +ADD readin/package-lock.json /tmp/package-lock.json RUN cd /tmp && npm ci RUN mkdir -p /opt/app && cp -a /tmp/node_modules /tmp/package.json /opt/app/ -WORKDIR /opt/app -ADD . /opt/app -ADD crontab.tmp /opt/app +ADD common/*.js /opt/app/common/ +ADD readin/*.js /opt/app/readin/ +ADD readin/package.json readin/crontab.tmp /opt/app/readin/ +WORKDIR /opt/app/readin RUN mkdir -p data #RUN apk add busybox-initscripts @@ -23,6 +29,3 @@ RUN touch cmds.sh \ && echo 'crond -f' >>cmds.sh CMD sh ./cmds.sh - - - diff --git a/readin/crontab.tmp b/readin/crontab.tmp index 60f118f..7fa4d2e 100644 --- a/readin/crontab.tmp +++ b/readin/crontab.tmp @@ -1,2 +1 @@ -3-59/5 * * * * cd /opt/app && npm start - +3-59/5 * * * * cd /opt/app/readin && npm start diff --git a/readin/deploy.sh b/readin/deploy.sh index dbe3a12..c785159 100755 --- a/readin/deploy.sh +++ b/readin/deploy.sh @@ -5,15 +5,20 @@ set -e # Angepasst aus /Projekte/Logbuch/deploy.sh # Unterschiede: nur linux/amd64, nativer Build auf esprimo (amd64) -> kein buildx/qemu, # eigener Dockerfile-Name (Dockerfile_readin). +# Build-Kontext ist das Repository-Wurzelverzeichnis, weil readin und +# readarchive sich die Module unter common/ teilen. REGISTRY="docker.citysensor.de" IMAGE_NAME="readin" TAG="${1:-latest}" PLATFORM="linux/amd64" -DOCKERFILE="Dockerfile_readin" +DOCKERFILE="readin/Dockerfile_readin" FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}" BUILD_DATE=$(date +%d.%m.%Y) +# ins Repository-Wurzelverzeichnis wechseln (eine Ebene ueber diesem Skript) +cd "$(dirname "$0")/.." + echo "==========================================" echo "readin Deploy Script (amd64)" echo "==========================================" @@ -22,6 +27,7 @@ echo "Image: ${IMAGE_NAME}" echo "Tag: ${TAG}" echo "Platform: ${PLATFORM}" echo "Dockerfile: ${DOCKERFILE}" +echo "Kontext: $(pwd)" echo "Build-Datum: ${BUILD_DATE}" echo "==========================================" echo "" diff --git a/readin/fetchnewdata.js b/readin/fetchnewdata.js index d0567ea..256070d 100644 --- a/readin/fetchnewdata.js +++ b/readin/fetchnewdata.js @@ -16,12 +16,13 @@ const TYP = process.env.TYP || '' const STORE = (process.env.STORE || 'both').toLowerCase() // 'mongo' | 'influx' | 'both' -import { doReadfromAPI as readin, statistics } from './readdata.js' +import { doReadfromAPI as readin } from './readdata.js' import { constructDBaseEntries as parse} from './parse.js' -import * as mongo from './mongo.js' -import * as influx from './influx_post.js' -import { logit, logerror } from './logit.js' +import * as mongo from '../common/mongo.js' +import * as influx from '../common/influx_post.js' +import { logit, logerror } from '../common/logit.js' +import { statistics } from '../common/statistics.js' import { DateTime } from 'luxon' import fs from 'fs' import mod_getopt from 'posix-getopt' diff --git a/readin/influx_post.js b/readin/influx_post.js deleted file mode 100644 index 938b22f..0000000 --- a/readin/influx_post.js +++ /dev/null @@ -1,100 +0,0 @@ -// Access to influxDB vie HTTP - -import axios from 'axios' -import { logit, logerror } from './logit.js' -import { DateTime } from 'luxon' -import { statistics } from'./readdata.js' - -let DEVELOP = process.env.DEVELOP || 'false' - -let INFLUXHOST = process.env.INFLUXHOST || "localhost" -let INFLUXPORT = process.env.INFLUXPORT || 8086 -let INFLUXTOKEN = process.env.INFLUXTOKEN || 'empty' -let INFLUXDATABUCKET = process.env.INFLUXDATABUCKET || "sensor_data" -let INFLUXORG = process.env.INFLUXORG || "citysensor" - -const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${INFLUXORG}` -const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms` - -export const influxRead = async (query) => { - let start = DateTime.now() - let data = [] - try { - let ret = await axios({ - method: 'post', - url: INFLUXURL_READ, - data: query, - headers: { - Authorization: `Token ${INFLUXTOKEN}`, - Accept: 'application/csv', - 'Content-type': 'application/vnd.flux' - }, - timeout: 10000, - }) - if (ret.status != 200) { - logerror(`doReadfromAPI Status: ${ret.status}`) - } - data = ret.data - } catch (e) { - logerror(`doReadfromAPI ${e}`) - } - logit(`ReadIn-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`) - return data -} - - -export const influxWrite = async (data) => { - let start = DateTime.now() - let ret - logit(INFLUXURL_WRITE) - if (DEVELOP === 'true') { - logit(`Token: ${INFLUXTOKEN}`) - } - try { - ret = await axios({ - method: 'post', - url: INFLUXURL_WRITE, - data: data, - headers: { - Authorization: `Token ${INFLUXTOKEN}`, - Accept: 'application/json', - 'Content-Type': 'text/plain; charset=utf-8' - }, - timeout: 10000, - }) - if (ret.status != 204) { - logerror(`doWrite2API Status: ${ret.status}`) - } - } catch (e) { - logerror(`doWrite2API ${e}`) - } - let statname = `writeInfluxData[sensor_data]Time` - statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds - logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`) - return ret -} - -/* -async function main() { - let data = ` - pm,sid=140 P1=12,P2=13 - pm,sid=142 P1=42,P2=13 - pm,sid=143 P1=43,P2=13 - pm,sid=144 P1=44,P2=13 - thp,sid=141 temperature=23.5,humidity=48,pressure=998 - ` - let ret = await influxWrite(data) - process.exit() - - let query = `from(bucket:"sensor_data") -|> range(start: -1mo) -|> filter(fn: (r) => r._measurement == "pm") -|> filter(fn: (r) => r.sid == "140") -` - let erg = await influxRead(query) - console.log(erg) -} - - -main().catch(console.error) -*/ diff --git a/readin/logit.js b/readin/logit.js deleted file mode 100644 index 19d8b28..0000000 --- a/readin/logit.js +++ /dev/null @@ -1,12 +0,0 @@ -import { DateTime} from 'luxon' - -export function logit(str) { - let s = `${DateTime.now().toISO()} => ${str}`; - console.log(s); -} - -export function logerror(str) { - let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`; - console.log(s); -} - diff --git a/readin/parse.js b/readin/parse.js index 334d788..8084467 100644 --- a/readin/parse.js +++ b/readin/parse.js @@ -1,8 +1,8 @@ // import logit from './logit.js' import { DateTime } from 'luxon' -import * as mongo from './mongo.js' -import { statistics } from'./readdata.js' -import { logit, logerror } from './logit.js' +import * as mongo from '../common/mongo.js' +import { statistics } from'../common/statistics.js' +import { logit, logerror } from '../common/logit.js' let actualProps = [] let newProps = [] diff --git a/readin/readdata.js b/readin/readdata.js index 878776c..c8b0adc 100644 --- a/readin/readdata.js +++ b/readin/readdata.js @@ -1,12 +1,12 @@ import axios from 'axios' import * as fs from 'fs' -import { logit, logerror} from'./logit.js' +import { logit, logerror} from'../common/logit.js' +import { statistics } from '../common/statistics.js' import { DateTime } from 'luxon' const API_URL = 'https://api.sensor.community/static/v1/data.json'; // URL to API on 'luftdaten.info' const SAVE_NAME = './data/aktdata.json'; // filename for actual data let LIVE = process.env.LIVE || 'true' -export let statistics = {}; export const doReadfromAPI = async () => { logit(`LIVE = ${LIVE}`)