admin/laermsensor-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

readin/readarchive: gemeinsame Module, Stack-Konventionen

Browse Source 
mongo.js, influx_post.js und logit.js lagen in beiden Komponenten doppelt
und waren auseinandergelaufen - so ist der Zeitzonen-Fehler entstanden, den
readin seit jeher richtig loest und readFromcsv.js zwei Jahre lang nicht.
Sie liegen jetzt einmal unter common/.

Wo die Fassungen sich widersprachen:
- MONGOAUTH wird als String verglichen. Die readarchive-Fassung pruefte nur
  auf truthy, dadurch schaltete auch MONGOAUTH=false die Auth ein.
- writeDataArray(client, coll, data) nimmt den Collection-Namen direkt;
  readarchive baut ihn mit dem neuen dataCollName(styp).
- getallProperties liefert das Array selbst. Die readarchive-Fassung mit
  {error, properties} war dort ungenutzt.
- properties_collection heisst einheitlich property_coll.

Das Sammelobjekt statistics lag in readin/readdata.js und wurde von
mongo.js und influx_post.js importiert. Es liegt jetzt in
common/statistics.js, damit die gemeinsamen Module readin nicht kennen
muessen.

Der Build-Kontext beider Images ist dadurch das Repository-Wurzelverzeichnis;
das Layout im Image spiegelt das Repository, damit ../common/... unveraendert
aufgeht. Die Volume-Zeile von readin im Compose zieht deshalb auf
/opt/app/readin/data um.

readarchive folgt jetzt den Konventionen des Stacks: deploy.sh statt
build_and_copy.sh, Dockerfile_readarchive statt Dockerfile_rfcsv, und die
eigene docker-compose.yml entfaellt - der Dienst haengt als Profil "tools"
im Compose des Stacks und wird mit Parametern gestartet:

  docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29

Getestet: beide Images gebaut, beide gegen Testdatenbanken laufen lassen.
Nicht deployt - die Images in der Registry sind unveraendert.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Reinhard X. Fürst
2026-07-31 07:25:54 +00:00
parent 7c6692c040
commit bd5cf9978d
26 changed files with 304 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+10
View File
@@ -0,0 +1,10 @@
.git
**/node_modules
**/data
**/log
**/*.log
**/.env
noise
noisesensors
sensorapi
noise.tgz
readarchive/influx_post.js → common/influx_post.js
+7 -28
View File
@@ -1,7 +1,11 @@
// Access to influxDB vie HTTP
/* Zugriff auf InfluxDB per HTTP
Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv).
*/
import axios from 'axios'
import { logit, logerror } from './logit.js'
import { statistics } from './statistics.js'
import { DateTime } from 'luxon'
let INFLUXHOST = process.env.INFLUXHOST || "localhost"
@@ -14,7 +18,7 @@ const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${IN
const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms`
// `${e}` liefert bei einem AggregateError nur "AggregateError" - die einzelnen
// Verbindungsfehler (je einer pro aufgelöster IP) stecken in e.errors bzw. e.cause.errors
// Verbindungsfehler (je einer pro aufgeloester IP) stecken in e.errors bzw. e.cause.errors
function describeError(e) {
const sub = e.errors || e.cause?.errors
if (sub?.length) {
@@ -52,7 +56,6 @@ export const influxRead = async (query) => {
return data
}
// liefert true, wenn Influx die Daten uebernommen hat, sonst false
export const influxWrite = async (data) => {
let start = DateTime.now()
@@ -77,31 +80,7 @@ export const influxWrite = async (data) => {
} catch (e) {
logerror(`doWrite2API ${INFLUXURL_WRITE} ${describeError(e)}`)
}
statistics['writeInfluxData[sensor_data]Time'] = DateTime.now().diff(start, ['seconds']).toObject().seconds
logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
return ok
}
/*
async function main() {
let data = `
pm,sid=140 P1=12,P2=13
pm,sid=142 P1=42,P2=13
pm,sid=143 P1=43,P2=13
pm,sid=144 P1=44,P2=13
thp,sid=141 temperature=23.5,humidity=48,pressure=998
`
let ret = await influxWrite(data)
process.exit()
let query = `from(bucket:"sensor_data")
|> range(start: -1mo)
|> filter(fn: (r) => r._measurement == "pm")
|> filter(fn: (r) => r.sid == "140")
`
let erg = await influxRead(query)
console.log(erg)
}
main().catch(console.error)
*/
readarchive/utilities/logit.js → common/logit.js
-1
View File
@@ -13,4 +13,3 @@ export function logerror(str) {
let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`;
console.log(s);
}
readin/mongo.js → common/mongo.js
+142 -32
View File
@@ -1,8 +1,19 @@
/* Interface for MongoDB
Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv).
Zusammengefuehrt aus den beiden zuvor getrennten Fassungen; wo sie sich
unterschieden, gilt:
- MONGOAUTH wird als String verglichen. Die alte readarchive-Fassung
pruefte nur auf truthy, dadurch schaltete auch MONGOAUTH=false die
Authentifizierung ein.
- writeDataArray(client, coll, data) nimmt den Collection-Namen direkt.
readarchive baut ihn mit dataCollName(styp).
- getallProperties liefert das Array selbst (readin-Fassung). Die
readarchive-Fassung mit {error, properties} war dort ungenutzt.
*/
import { MongoClient } from 'mongodb'
import { logit, logerror } from './logit.js'
import { statistics } from './readdata.js'
import { statistics } from './statistics.js'
import { DateTime } from 'luxon'
let DEVELOP = process.env.DEVELOP || 'false'
@@ -13,13 +24,18 @@ let MONGOAUTH = process.env.MONGOAUTH || 'false'
let MONGOUSRP = process.env.MONGOUSRP || ''
let MONGOBASE = process.env.MONGOBASE || 'sensor_data'
let MONGO_URL = 'mongodb://' + MONGOHOST + ':' + MONGOPORT; // URL to mongo database
if (MONGOAUTH == 'true') {
MONGO_URL = 'mongodb://' + MONGOUSRP + '@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin'; // URL to mongo database
}
export const property_coll = 'properties'
const data_collection = 'sensors'
// Datencollection eines Sensortyps, z.B. dataCollName('noise') -> 'noise_sensors'
export const dataCollName = (styp) => {
return styp + '_' + data_collection
}
const addandshowstatistics = (client, text, field, start) => {
statistics[field] = DateTime.now().diff(start, ['seconds']).toObject().seconds
@@ -33,7 +49,7 @@ export const connectMongo = async () => {
} else {
logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`)
}
let client = await MongoClient.connect(MONGO_URL, { useNewUrlParser: true, useUnifiedTopology: true })
let client = await MongoClient.connect(MONGO_URL)
if ( DEVELOP === 'true') {
logit(`Mongodbase connected to ${MONGO_URL}`)
} else {
@@ -47,24 +63,107 @@ export const connectMongo = async () => {
}
export const closeMongo = async (client) => {
client.close()
try {
await client.close()
}
catch(error){
throw(error)
}
}
/* ***************************************************
// READ routines
******************************************************/
// Read properties from the database - oeffnet und schliesst selbst
export const readProperties = async (query, limit = 0) => {
let ret = {err: null, properties: null}
let client = await connectMongo()
try {
if ("sid" in query) { // if sid is given, read property for sid
ret.properties = await client.db(MONGOBASE).collection(property_coll).findOne({_id: query.sid})
} else { // otherwise read props corresponding to query
ret.properties = await client.db(MONGOBASE).collection(property_coll).find(query).limit(limit).toArray()
}
} catch (e) {
ret.err = e
}
finally {
client.close()
}
return ret
}
export const getallProperties = async (client) => {
return await client.db(MONGOBASE).collection(property_coll)
.find().sort({ _id: 1 }).toArray()
}
export const checkOneproperty = async (client, sid) => {
return await client.db(MONGOBASE).collection("properties")
return await client.db(MONGOBASE).collection(property_coll)
.findOne({ _id: sid })
}
export const getOneproperty = async (client, sid) => {
let ret = {error: false, errortext: '', property: null}
try {
ret.property = await client.db(MONGOBASE).collection(property_coll)
.findOne({_id: sid})
} catch (e) {
ret = {error: true, errortext: e}
}
return ret
}
export const getOneSensorOneday = async (client, sid, day, styp) => {
let ret = {error: false, errortext: '', date: []}
// muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC),
// sonst passt das Suchfenster nicht auf die Daten des Tages
let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' })
let start = d.startOf('day').toJSDate()
let end = d.startOf('day').plus({day:1}).toJSDate()
try {
let erg = await client.db(MONGOBASE).collection(dataCollName(styp))
.find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray()
ret.data = erg
} catch(e) {
ret.error = true
ret.errortext = e
}
return ret
}
export const getLocationIDs = async (client, stype) => {
let ret = {error: false, errortext: '', locations: []}
try {
ret.locations = await client.db(MONGOBASE).collection(property_coll)
.distinct('location.0.id', {type: stype})
} catch (e) {
ret.error = true
ret.errortext = e
}
return ret
}
export const getTHPSensors = async (client, lid) => {
let ret = {error: false, errortext: '', sensors: []}
try {
ret.sensors = await client.db(MONGOBASE).collection(property_coll)
.findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}})
} catch (e) {
ret.error = true
ret.errortext = e
}
return ret
}
/* ***************************************************
// WRITE routines
******************************************************/
export const writeOneproperty = async (client, prop) => {
try {
let result = await client.db(MONGOBASE).collection("properties")
let result = await client.db(MONGOBASE).collection(property_coll)
.insertOne(prop)
} catch (e) {
if (e.code == 11000) {
@@ -80,7 +179,7 @@ export const writeProperties = async (client, props) => {
let result
let startAll = DateTime.now();
let start
let coll = client.db(MONGOBASE).collection("properties")
let coll = client.db(MONGOBASE).collection(property_coll)
if (props.new.length !== 0) {
start = DateTime.now();
try {
@@ -120,6 +219,19 @@ export const writeProperties = async (client, props) => {
addandshowstatistics(client, 'properties', 'writePropsTime', startAll)
}
export const writeOneSensor = async (client, data, styp) => {
let ret = {error: false, errortext: '', inserted: 1}
try {
await client.db(MONGOBASE).collection(dataCollName(styp))
.insertOne(data)
} catch (e) {
ret.error = true
ret.errortext = e
ret.inserted = 0
}
return ret
}
export const writeDataArray = async (client, coll, data) => {
let result
let start = DateTime.now();
@@ -131,11 +243,7 @@ export const writeDataArray = async (client, coll, data) => {
console.error(e)
}
}
let statname = `writeMongoData[${coll}]Time`
addandshowstatistics(client, `${data.length} entries for ${coll}`, `writeMongoData[${coll}]Time`, start)
// statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
// logit(`Write Data for ${coll} to mongoDB: Time: ${statistics[statname]} sec.`)
}
export const writeStatistic = async (client, stat) => {
@@ -151,27 +259,17 @@ export const writeStatistic = async (client, stat) => {
addandshowstatistics(client, `statistics`, `writeStatisticTime`, start)
}
export const dropColl = async (client, coll) => {
export const bulkWrite = async (client, coll, data) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection(coll).drop()
result = await client.db(MONGOBASE).collection(coll)
.bulkWrite(data, { ordered: false })
} catch (e) {
console.error(e)
}
logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
}
export const createIndex = async (client, coll) => {
let result
let start = DateTime.now()
try {
result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" })
} catch (e) {
console.error(e)
}
logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start)
return result
}
export const bulkUpdateMapdata = async (client, data) => {
@@ -187,16 +285,28 @@ export const bulkUpdateMapdata = async (client, data) => {
return result
}
/* ***************************************************
// Wartung
******************************************************/
export const bulkWrite = async (client, coll, data) => {
export const dropColl = async (client, coll) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection(coll)
.bulkWrite(data, { ordered: false })
result = await client.db(MONGOBASE).collection(coll).drop()
} catch (e) {
console.error(e)
}
addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start)
return result
logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
}
export const createIndex = async (client, coll) => {
let result
let start = DateTime.now()
try {
result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" })
} catch (e) {
console.error(e)
}
logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
}
common/statistics.js
+7
View File
@@ -0,0 +1,7 @@
/* Gemeinsames Sammelobjekt fuer Laufzeit-Statistiken.
Lag frueher in readin/readdata.js. Liegt hier, damit auch die gemeinsam
genutzten Module darauf schreiben koennen, ohne readin kennen zu muessen.
*/
export const statistics = {}
noisesensors/docker-compose.yml
+23 -3
View File
@@ -43,10 +43,30 @@ services:
INFLUXTOKEN: ${DOCKER_INFLUXTOKEN}
volumes:
- ${LOCALDIR}/noisesensors/log:/var/log
- ${LOCALDIR}/noisesensors/data:/opt/app/data
container_name: readin
# Pfad im Image ist jetzt /opt/app/readin, weil readin und readarchive
# sich common/ teilen und das Layout das Repository spiegelt
- ${LOCALDIR}/noisesensors/data:/opt/app/readin/data
container_name: readin
restart: unless-stopped
# Kein Dauerdienst, sondern ein Lauf mit Parametern - liest rueckwirkend die
# Tagesdateien von archive.sensor.community ein:
# docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29
# Das Profil verhindert, dass "docker compose up" ihn mitstartet.
readarchive:
image: docker.citysensor.de/readarchive
profiles: ["tools"]
environment:
MONGOHOST: mongodb
INFLUXHOST: influxdb
MONGOAUTH: "true"
MONGOUSRP: ${MONGO_ROOT_USERNAME}:${MONGO_ROOT_PASSWORD}
INFLUXTOKEN: ${DOCKER_INFLUXTOKEN}
volumes:
- ${LOCALDIR}/noisesensors/log:/var/log
- ${LOCALDIR}/noisesensors/data/readarchive:/opt/app/readarchive/data
restart: "no"
logrotate:
image: linkyard/logrotate
volumes:
readarchive/Dockerfile_rfcsv → readarchive/Dockerfile_readarchive
+14 -10
View File
@@ -1,15 +1,21 @@
FROM node:22-alpine
ADD package.json package-lock.json /tmp/
# Build-Kontext ist das Repository-Wurzelverzeichnis (siehe deploy.sh), weil
# readarchive und readin sich die Module unter common/ teilen. Das Layout im
# Image spiegelt das Repository, damit die relativen Importe (../common/...)
# unveraendert funktionieren.
ADD readarchive/package.json readarchive/package-lock.json /tmp/
RUN cd /tmp && npm ci --omit=dev
RUN mkdir -p /opt/app && cp -a /tmp/node_modules /tmp/package.json /opt/app/
WORKDIR /opt/app
ADD *.js /opt/app/
RUN mkdir /opt/app/utilities
ADD utilities/*.js /opt/app/utilities
ADD common/*.js /opt/app/common/
ADD readarchive/*.js /opt/app/readarchive/
ADD readarchive/package.json /opt/app/readarchive/
ADD readarchive/utilities/*.js /opt/app/readarchive/utilities/
# readFromcsv.js legt hier die Sensorliste des Tages ab (data/list.json)
RUN mkdir -p /opt/app/data
RUN mkdir -p /opt/app/readarchive/data
# betrifft nur noch die Uhrzeit in den Log-Zeilen - die Messwerte selbst
# werden seit dem UTC-Fix unabhaengig von der Zeitzone der Maschine gespeichert
@@ -17,11 +23,9 @@ RUN apk add --no-cache tzdata
ENV TZ=Europe/Berlin
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
WORKDIR /opt/app/readarchive
# Exec-Form: nur so werden Argumente aus "docker run <image> ..." an das
# Programm durchgereicht. CMD ist der Default, den eigene Argumente ersetzen.
#
# docker run --rm --env-file .env -e INFLUXHOST=influxdb -e MONGOHOST=mongo \
# --network laerm_readfromcsv_to_database_default rfcsv -t noise -s 2026-07-25
#
ENTRYPOINT ["node", "./readFromcsv.js"]
CMD ["-t", "noise"]
readarchive/build_and_copy.sh
-63
View File
@@ -1,63 +0,0 @@
#!/bin/bash
# Build Docker-Container
#
# Call: buildit.sh name [target]
#
# The Dockerfile must be named like Dockerfile_name
#
# 2018-09-20 rxf
# - before sending docker image to remote, tag actual remote image
#
# 2018-09-14 rxf
# - first Version
#
set -x
port=""
orgName=rfcsv
name=rfcsv
usage()
{
echo "Usage build_and_copy.sh [-p port] [-n name] target"
echo " Build docker container $name and copy to target"
echo "Params:"
echo " target: Where to copy the container to "
echo " -p port: ssh port (default 22)"
echo " -n name: new name for container (default: $orgName)"
}
while getopts n:p:h? o
do
case "$o" in
n) name="$OPTARG";;
p) port="-p $OPTARG";;
h) usage; exit 0;;
*) usage; exit 1;;
esac
done
shift $((OPTIND-1))
while [ $# -gt 0 ]; do
if [[ -z "$target" ]]; then
target=$1
shift
else
echo "bad option $1"
# exit 1
shift
fi
done
docker build -f Dockerfile_$orgName -t $name .
dat=`date +%Y%m%d%H%M`
if [ "$target" == "localhost" ]
then
docker tag $name $name:V_$dat
exit
fi
ssh $port $target "docker tag $name $name:V_$dat"
docker save $name | bzip2 | pv | ssh $port $target 'bunzip2 | docker load'
readarchive/crontab.tmp
-2
View File
@@ -1,2 +0,0 @@
#33 10 * * * cd /opt/app && node ./readFromcsv.js -t laerm >>/var/log/readFrom.log 2>&1
#53 10 * * * cd /opt/app && node ./readFromcsv.js -t radia >>/var/log/readFrom.log 2>&1
readarchive/deploy.sh
Executable
+63
View File
@@ -0,0 +1,63 @@
#!/bin/bash
set -e
# Angepasst aus readin/deploy.sh
# Unterschied: Build-Kontext ist das Repository-Wurzelverzeichnis, weil
# readarchive und readin sich die Module unter common/ teilen.
REGISTRY="docker.citysensor.de"
IMAGE_NAME="readarchive"
TAG="${1:-latest}"
PLATFORM="linux/amd64"
DOCKERFILE="readarchive/Dockerfile_readarchive"
FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}"
BUILD_DATE=$(date +%d.%m.%Y)
# ins Repository-Wurzelverzeichnis wechseln (eine Ebene ueber diesem Skript)
cd "$(dirname "$0")/.."
echo "=========================================="
echo "readarchive Deploy Script (amd64)"
echo "=========================================="
echo "Registry: ${REGISTRY}"
echo "Image: ${IMAGE_NAME}"
echo "Tag: ${TAG}"
echo "Platform: ${PLATFORM}"
echo "Dockerfile: ${DOCKERFILE}"
echo "Kontext: $(pwd)"
echo "Build-Datum: ${BUILD_DATE}"
echo "=========================================="
echo ""
# Hinweis: Login wird uebersprungen, weil die Credentials fuer ${REGISTRY}
# bereits in ~/.docker/config.json liegen (Container laufen bereits aus dieser Registry).
# Bei Bedarf manuell: docker login ${REGISTRY}
echo ">>> Baue Image (nur ${PLATFORM}, nativ) ..."
docker build \
--platform "${PLATFORM}" \
-f "${DOCKERFILE}" \
-t "${FULL_IMAGE}" \
.
echo ""
echo ">>> Pushe ${FULL_IMAGE} ..."
docker push "${FULL_IMAGE}"
echo ""
# Falls ein anderer Tag als 'latest' gebaut wurde, zusaetzlich latest setzen+pushen
if [ "${TAG}" != "latest" ]; then
echo ">>> Tagge zusaetzlich als :latest ..."
docker tag "${FULL_IMAGE}" "${REGISTRY}/${IMAGE_NAME}:latest"
docker push "${REGISTRY}/${IMAGE_NAME}:latest"
echo ""
fi
echo "=========================================="
echo "Build + Push fertig: ${FULL_IMAGE}"
echo "=========================================="
echo ""
echo "readarchive ist kein Dauerdienst, sondern ein Lauf mit Parametern:"
echo " cd /opt/stacks/esprimo/noisesensors"
echo " docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29"
readarchive/docker-compose.yml
-56
View File
@@ -1,56 +0,0 @@
version: "3.9"
volumes:
mongo_vol:
influx_vol:
services:
# rfcsv:
# image: rfcsv
# volumes:
# - /var/log/rfcsv:/var/log
# - /home/rxf/CERTS:certs
# environment:
# - MONGOHOST=207.180.224.98
# - MONGOPORT=20019
# #- MONGOAUTH=true
# #- "MONGOUSRP=rxf:5C5dB|m"
# - MONGOBASE=allsensors
# - TZ=Europe/Berlin
# container_name: rfcsv
# restart: always
mongodb:
image: mongo
volumes:
- mongo_vol:/data/db
ports:
- "27017:27017"
container_name: mongo
#environment:
# - MONGO_INITDB_ROOT_USERNAME=${MONGO_ROOT_USER}
# - MONGO_INITDB_ROOT_PASSWORD=${MONGO_ROOT_PASSWD}
#command: '--auth'
restart: unless-stopped
influx:
image: influxdb:2.7
ports:
- '8086:8086'
volumes:
# /etc/influxdb2 NICHT mounten: das Entrypoint legt dort beim Setup das
# CLI-Profil "default" an. Ein vorhandenes Profil laesst den Setup
# scheitern - und danach loescht das Entrypoint bolt+engine im Volume.
- influx_vol:/var/lib/influxdb2
environment:
# Werte kommen aus .env, damit App und Container dieselben benutzen
- DOCKER_INFLUXDB_INIT_MODE=setup
- DOCKER_INFLUXDB_INIT_USERNAME=${DOCKER_INFLUXDB_INIT_USERNAME}
- DOCKER_INFLUXDB_INIT_PASSWORD=${DOCKER_INFLUXDB_INIT_PASSWORD}
- DOCKER_INFLUXDB_INIT_ORG=${DOCKER_INFLUXDB_INIT_ORG}
- DOCKER_INFLUXDB_INIT_BUCKET=${DOCKER_INFLUXDB_INIT_BUCKET}
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${INFLUXTOKEN}
restart:
unless-stopped
container_name: influxdb
readarchive/logit.js
-12
View File
@@ -1,12 +0,0 @@
import { DateTime} from 'luxon'
export function logit(str) {
let s = `${DateTime.now().toISO()} => ${str}`;
console.log(s);
}
export function logerror(str) {
let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`;
console.log(s);
}
readarchive/mongo.js
-197
View File
@@ -1,197 +0,0 @@
/* Interface for MongoDB
*/
import { MongoClient } from 'mongodb'
import { logit, logerror } from './utilities/logit.js'
import { DateTime } from 'luxon'
import {returnOnError} from "./utilities/reporterror.js";
const DEVELOP = process.env.DEVELOP || false;
const MONGOHOST = process.env.MONGOHOST || 'localhost';
const MONGOPORT = process.env.MONGOPORT || 27017;
const MONGOAUTH = process.env.MONGOAUTH || false;
const MONGOUSRP = process.env.MONGOUSRP || "";
const MONGOBASE = process.env.MONGOBASE || 'sensor_data';
const MONGO_URL = MONGOAUTH ? 'mongodb://'+MONGOUSRP+'@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin' : 'mongodb://'+MONGOHOST+':'+MONGOPORT; // URL to mongo database
if (DEVELOP) {
console.log(`MongoURL = "${MONGO_URL}" and Database = ${MONGOBASE}`);
}
const statistics = {}
export const properties_collection = 'properties'
const data_collection = 'sensors'
export const connectMongo = async () => {
try {
if(DEVELOP === 'true') {
logit(`Try to connect to ${MONGO_URL}`)
} else {
logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`)
}
let client = await MongoClient.connect(MONGO_URL)
if ( DEVELOP === 'true') {
logit(`Mongodbase connected to ${MONGO_URL}`)
} else {
logit('Mongodbase connected')
}
return client
}
catch (error) {
throw (error)
}
}
export const closeMongo = async (client) => {
try {
await client.close()
}
catch(error){
throw(error)
}
}
const listDatabases = async (client) => {
let databasesList = await client.db().admin().listDatabases();
console.log("Databases:");
databasesList.databases.forEach(db => console.log(` - ${db.name}`));
}
/* ***************************************************
// READ routines
******************************************************/
// Read properties from the database
export const readProperties = async (query, limit = 0) => {
let ret = {err: null, properties: null}
let client = await connectMongo()
try {
if ("sid" in query) { // if sid is given, read property for sid
ret.properties = await client.db(MONGOBASE).collection(properties_collection).findOne({_id: query.sid})
} else { // otherwise read props corresponding to query
ret.properties = await client.db(MONGOBASE).collection(properties_collection).find(query).limit(limit).toArray()
}
} catch (e) {
ret.err = e
}
finally {
client.close()
}
return ret
}
export const getallProperties = async (client) => {
let ret = {error: false, errortext: '', properties: []}
try
{
ret.properties = await client.db(MONGOBASE).collection(properties_collection)
.find().sort({_id: 1}).toArray()
}
catch(e) {
ret = {error: true, errortext: e}
}
return ret
}
export const getOneproperty = async (client, sid) => {
let ret = {error: false, errortext: '', property: null}
try {
ret.property = await client.db(MONGOBASE).collection(properties_collection)
.findOne({_id: sid})
} catch (e) {
ret = {error: true, errortext: e}
}
return ret
}
const getCollName = (coll, st) => {
return st + '_' + coll
}
export const getOneSensorOneday = async (client, sid, day, styp) => {
let ret = {error: false, errortext: '', date: []}
// muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC),
// sonst passt das Suchfenster nicht auf die Daten des Tages
let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' })
let start = d.startOf('day').toJSDate()
let end = d.startOf('day').plus({day:1}).toJSDate()
try {
let erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
.find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray()
ret.data = erg
} catch(e) {
ret.error = true
ret.errortext = e
}
return ret
}
export const writeOneSensor = async (client, data, styp) => {
let ret = {error: false, errortext: '', inserted: 1}
let erg
try {
erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
.insertOne(data)
} catch (e) {
ret.error = true
ret.errortext = e
ret.inserted = 0
}
return ret
}
export const writeDataArray = async (client, data, styp) => {
let result
let start = DateTime.now();
try {
result = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
.insertMany(data, {ordered: false})
} catch (e) {
if(e.code !== 11000) {
console.error(e)
}
}
let statname = `writeData${getCollName(data_collection, styp)}Time`
statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
logit(`Write Data for ${getCollName(data_collection, styp)} to mongoDB: Time: ${statistics[statname]} sec.`)
}
export const bulkWrite = async (client, coll, data) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection(coll)
.bulkWrite(data, {ordered: false})
} catch (e) {
console.error(e)
}
logit(`Write Properties: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
return result
}
export const getLocationIDs = async (client, stype) => {
let ret = {error: false, errortext: '', locations: []}
try {
ret.locations = await client.db(MONGOBASE).collection(properties_collection)
.distinct('location.0.id', {type: stype})
} catch (e) {
ret.error = true
ret.errortext = e
}
return ret
}
export const getTHPSensors = async (client, lid) => {
let ret = {error: false, errortext: '', sensors: []}
try {
ret.sensors = await client.db(MONGOBASE).collection(properties_collection)
.findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}})
} catch (e) {
ret.error = true
ret.errortext = e
}
return ret
}
readarchive/mqtt.js
+1 -1
View File
@@ -1,7 +1,7 @@
// MQTT interface
import * as mqtt from 'mqtt'
import { logit, logerror } from './logit.js'
import { logit, logerror } from '../common/logit.js'
const MQTTHOST = process.env.MQTTHOST || 'rexfue.de'
const MQTTPORT = process.env.MQTTPORT || 1883
readarchive/readFromcsv.js
+5 -5
View File
@@ -37,13 +37,13 @@ const MAXTRIES = 10
import 'dotenv/config'
import axios from 'axios'
import https from 'https'
import * as mongo from './mongo.js'
import * as influx from './influx_post.js'
import * as mongo from '../common/mongo.js'
import * as influx from '../common/influx_post.js'
import fs from 'fs'
import csv from 'csvtojson'
import pkg from './package.json' with { type: "json" }
import mod_getopt from 'posix-getopt'
import { logit, logerror } from './utilities/logit.js'
import { logit, logerror } from '../common/logit.js'
import { DateTime, Duration } from 'luxon'
import { checkProperties} from "./utilities/checkprops.js";
import { version } from "os"
@@ -376,7 +376,7 @@ async function enterOneSensorinDB(client, sid, erg, dataline, day, styp, args) {
}
if((args.database === 'both') || (args.database === 'mongo')) {
if(erg.length !== 0) {
await mongo.writeDataArray(client, erg, styp)
await mongo.writeDataArray(client, mongo.dataCollName(styp), erg)
}
}
return erg.length
@@ -514,7 +514,7 @@ async function main() {
logit(`Inserted: ${inserted}`)
// enter changed properties into database
if(newProps.length > 0) {
await mongo.bulkWrite(client, mongo.properties_collection, newProps)
await mongo.bulkWrite(client, mongo.property_coll, newProps)
}
await mongo.closeMongo(client)
if (failedInfluxWrites.length > 0) {
readarchive/utilities/checkprops.js
+2 -2
View File
@@ -1,7 +1,7 @@
import { DateTime } from 'luxon'
import { logit, logerror } from './logit.js'
import { logit, logerror } from '../../common/logit.js'
import { returnOnError } from './reporterror.js'
import * as mongo from '../mongo.js'
import * as mongo from '../../common/mongo.js'
// Check lat/lon and convert to float
function checkLatLon(w) {
readarchive/utilities/reporterror.js
+1 -1
View File
@@ -1,4 +1,4 @@
import {logit} from "./logit.js";
import {logit} from "../../common/logit.js";
export const reportError = (message, errortext) => {
message.error = true
readin/.dockerignore
-10
View File
@@ -1,10 +0,0 @@
node-modules
.gitignore
.dockerignore
build_and_copy.sh
docker-compose.yml
Dockerfile*
INFLUXSERVER
TRANSFERE
data
readin/Dockerfile_readin
+11 -8
View File
@@ -1,14 +1,20 @@
FROM node:22-alpine
ADD package.json /tmp/package.json
ADD package-lock.json /tmp/package-lock.json
# Build-Kontext ist das Repository-Wurzelverzeichnis (siehe deploy.sh), weil
# readin und readarchive sich die Module unter common/ teilen. Das Layout im
# Image spiegelt das Repository, damit die relativen Importe (../common/...)
# unveraendert funktionieren.
ADD readin/package.json /tmp/package.json
ADD readin/package-lock.json /tmp/package-lock.json
RUN cd /tmp && npm ci
RUN mkdir -p /opt/app && cp -a /tmp/node_modules /tmp/package.json /opt/app/
WORKDIR /opt/app
ADD . /opt/app
ADD crontab.tmp /opt/app
ADD common/*.js /opt/app/common/
ADD readin/*.js /opt/app/readin/
ADD readin/package.json readin/crontab.tmp /opt/app/readin/
WORKDIR /opt/app/readin
RUN mkdir -p data
#RUN apk add busybox-initscripts
@@ -23,6 +29,3 @@ RUN touch cmds.sh \
&& echo 'crond -f' >>cmds.sh
CMD sh ./cmds.sh
readin/crontab.tmp
+1 -2
View File
@@ -1,2 +1 @@
3-59/5 * * * * cd /opt/app && npm start
3-59/5 * * * * cd /opt/app/readin && npm start
readin/deploy.sh
+7 -1
View File
@@ -5,15 +5,20 @@ set -e
# Angepasst aus /Projekte/Logbuch/deploy.sh
# Unterschiede: nur linux/amd64, nativer Build auf esprimo (amd64) -> kein buildx/qemu,
# eigener Dockerfile-Name (Dockerfile_readin).
# Build-Kontext ist das Repository-Wurzelverzeichnis, weil readin und
# readarchive sich die Module unter common/ teilen.
REGISTRY="docker.citysensor.de"
IMAGE_NAME="readin"
TAG="${1:-latest}"
PLATFORM="linux/amd64"
DOCKERFILE="Dockerfile_readin"
DOCKERFILE="readin/Dockerfile_readin"
FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}"
BUILD_DATE=$(date +%d.%m.%Y)
# ins Repository-Wurzelverzeichnis wechseln (eine Ebene ueber diesem Skript)
cd "$(dirname "$0")/.."
echo "=========================================="
echo "readin Deploy Script (amd64)"
echo "=========================================="
@@ -22,6 +27,7 @@ echo "Image: ${IMAGE_NAME}"
echo "Tag: ${TAG}"
echo "Platform: ${PLATFORM}"
echo "Dockerfile: ${DOCKERFILE}"
echo "Kontext: $(pwd)"
echo "Build-Datum: ${BUILD_DATE}"
echo "=========================================="
echo ""
readin/fetchnewdata.js
+5 -4
View File
@@ -16,12 +16,13 @@
const TYP = process.env.TYP || ''
const STORE = (process.env.STORE || 'both').toLowerCase() // 'mongo' | 'influx' | 'both'
import { doReadfromAPI as readin, statistics } from './readdata.js'
import { doReadfromAPI as readin } from './readdata.js'
import { constructDBaseEntries as parse} from './parse.js'
import * as mongo from './mongo.js'
import * as influx from './influx_post.js'
import { logit, logerror } from './logit.js'
import * as mongo from '../common/mongo.js'
import * as influx from '../common/influx_post.js'
import { logit, logerror } from '../common/logit.js'
import { statistics } from '../common/statistics.js'
import { DateTime } from 'luxon'
import fs from 'fs'
import mod_getopt from 'posix-getopt'
readin/influx_post.js
-100
View File
@@ -1,100 +0,0 @@
// Access to influxDB vie HTTP
import axios from 'axios'
import { logit, logerror } from './logit.js'
import { DateTime } from 'luxon'
import { statistics } from'./readdata.js'
let DEVELOP = process.env.DEVELOP || 'false'
let INFLUXHOST = process.env.INFLUXHOST || "localhost"
let INFLUXPORT = process.env.INFLUXPORT || 8086
let INFLUXTOKEN = process.env.INFLUXTOKEN || 'empty'
let INFLUXDATABUCKET = process.env.INFLUXDATABUCKET || "sensor_data"
let INFLUXORG = process.env.INFLUXORG || "citysensor"
const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${INFLUXORG}`
const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms`
export const influxRead = async (query) => {
let start = DateTime.now()
let data = []
try {
let ret = await axios({
method: 'post',
url: INFLUXURL_READ,
data: query,
headers: {
Authorization: `Token ${INFLUXTOKEN}`,
Accept: 'application/csv',
'Content-type': 'application/vnd.flux'
},
timeout: 10000,
})
if (ret.status != 200) {
logerror(`doReadfromAPI Status: ${ret.status}`)
}
data = ret.data
} catch (e) {
logerror(`doReadfromAPI ${e}`)
}
logit(`ReadIn-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
return data
}
export const influxWrite = async (data) => {
let start = DateTime.now()
let ret
logit(INFLUXURL_WRITE)
if (DEVELOP === 'true') {
logit(`Token: ${INFLUXTOKEN}`)
}
try {
ret = await axios({
method: 'post',
url: INFLUXURL_WRITE,
data: data,
headers: {
Authorization: `Token ${INFLUXTOKEN}`,
Accept: 'application/json',
'Content-Type': 'text/plain; charset=utf-8'
},
timeout: 10000,
})
if (ret.status != 204) {
logerror(`doWrite2API Status: ${ret.status}`)
}
} catch (e) {
logerror(`doWrite2API ${e}`)
}
let statname = `writeInfluxData[sensor_data]Time`
statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
return ret
}
/*
async function main() {
let data = `
pm,sid=140 P1=12,P2=13
pm,sid=142 P1=42,P2=13
pm,sid=143 P1=43,P2=13
pm,sid=144 P1=44,P2=13
thp,sid=141 temperature=23.5,humidity=48,pressure=998
`
let ret = await influxWrite(data)
process.exit()
let query = `from(bucket:"sensor_data")
|> range(start: -1mo)
|> filter(fn: (r) => r._measurement == "pm")
|> filter(fn: (r) => r.sid == "140")
`
let erg = await influxRead(query)
console.log(erg)
}
main().catch(console.error)
*/
readin/logit.js
-12
View File
@@ -1,12 +0,0 @@
import { DateTime} from 'luxon'
export function logit(str) {
let s = `${DateTime.now().toISO()} => ${str}`;
console.log(s);
}
export function logerror(str) {
let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`;
console.log(s);
}
readin/parse.js
+3 -3
View File
@@ -1,8 +1,8 @@
// import logit from './logit.js'
import { DateTime } from 'luxon'
import * as mongo from './mongo.js'
import { statistics } from'./readdata.js'
import { logit, logerror } from './logit.js'
import * as mongo from '../common/mongo.js'
import { statistics } from'../common/statistics.js'
import { logit, logerror } from '../common/logit.js'
let actualProps = []
let newProps = []
readin/readdata.js
+2 -2
View File
@@ -1,12 +1,12 @@
import axios from 'axios'
import * as fs from 'fs'
import { logit, logerror} from'./logit.js'
import { logit, logerror} from'../common/logit.js'
import { statistics } from '../common/statistics.js'
import { DateTime } from 'luxon'
const API_URL = 'https://api.sensor.community/static/v1/data.json'; // URL to API on 'luftdaten.info'
const SAVE_NAME = './data/aktdata.json'; // filename for actual data
let LIVE = process.env.LIVE || 'true'
export let statistics = {};
export const doReadfromAPI = async () => {
logit(`LIVE = ${LIVE}`)