readin/readarchive: gemeinsame Module, Stack-Konventionen

mongo.js, influx_post.js und logit.js lagen in beiden Komponenten doppelt und waren auseinandergelaufen - so ist der Zeitzonen-Fehler entstanden, den readin seit jeher richtig loest und readFromcsv.js zwei Jahre lang nicht. Sie liegen jetzt einmal unter common/. Wo die Fassungen sich widersprachen: - MONGOAUTH wird als String verglichen. Die readarchive-Fassung pruefte nur auf truthy, dadurch schaltete auch MONGOAUTH=false die Auth ein. - writeDataArray(client, coll, data) nimmt den Collection-Namen direkt; readarchive baut ihn mit dem neuen dataCollName(styp). - getallProperties liefert das Array selbst. Die readarchive-Fassung mit {error, properties} war dort ungenutzt. - properties_collection heisst einheitlich property_coll. Das Sammelobjekt statistics lag in readin/readdata.js und wurde von mongo.js und influx_post.js importiert. Es liegt jetzt in common/statistics.js, damit die gemeinsamen Module readin nicht kennen muessen. Der Build-Kontext beider Images ist dadurch das Repository-Wurzelverzeichnis; das Layout im Image spiegelt das Repository, damit ../common/... unveraendert aufgeht. Die Volume-Zeile von readin im Compose zieht deshalb auf /opt/app/readin/data um. readarchive folgt jetzt den Konventionen des Stacks: deploy.sh statt build_and_copy.sh, Dockerfile_readarchive statt Dockerfile_rfcsv, und die eigene docker-compose.yml entfaellt - der Dienst haengt als Profil "tools" im Compose des Stacks und wird mit Parametern gestartet: docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29 Getestet: beide Images gebaut, beide gegen Testdatenbanken laufen lassen. Nicht deployt - die Images in der Registry sind unveraendert. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 07:25:54 +00:00
parent 7c6692c040
commit bd5cf9978d
26 changed files with 304 additions and 555 deletions
@@ -0,0 +1,86 @@
+/* Zugriff auf InfluxDB per HTTP
+
+Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv).
+*/
+
+import axios from 'axios'
+import { logit, logerror } from './logit.js'
+import { statistics } from './statistics.js'
+import { DateTime } from 'luxon'
+
+let INFLUXHOST = process.env.INFLUXHOST || "localhost"
+let INFLUXPORT = process.env.INFLUXPORT || 8086
+let INFLUXTOKEN = process.env.INFLUXTOKEN || 'empty'
+let INFLUXDATABUCKET = process.env.INFLUXDATABUCKET || "sensor_data"
+let INFLUXORG = process.env.INFLUXORG || "citysensor"
+
+const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${INFLUXORG}`
+const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms`
+
+// `${e}` liefert bei einem AggregateError nur "AggregateError" - die einzelnen
+// Verbindungsfehler (je einer pro aufgeloester IP) stecken in e.errors bzw. e.cause.errors
+function describeError(e) {
+    const sub = e.errors || e.cause?.errors
+    if (sub?.length) {
+        return `${e.message || e.name}: ` + sub.map(s => `${s.code} ${s.address}:${s.port}`).join(', ')
+    }
+    if (e.response) {
+        return `HTTP ${e.response.status} ${JSON.stringify(e.response.data)}`
+    }
+    return `${e.code ? e.code + ' ' : ''}${e.message || e}`
+}
+
+export const influxRead = async (query) => {
+    let start = DateTime.now()
+    let data = []
+    try {
+        let ret = await axios({
+            method: 'post',
+            url: INFLUXURL_READ,
+            data: query,
+            headers: {
+                Authorization: `Token ${INFLUXTOKEN}`,
+                Accept: 'application/csv',
+                'Content-type': 'application/vnd.flux'
+            },
+            timeout: 10000,
+        })
+        if (ret.status != 200) {
+            logerror(`doReadfromAPI Status: ${ret.status}`)
+        }
+        data = ret.data
+    } catch (e) {
+        logerror(`doReadfromAPI  ${INFLUXURL_READ}  ${describeError(e)}`)
+    }
+    logit(`ReadIn-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
+    return data
+}
+
+// liefert true, wenn Influx die Daten uebernommen hat, sonst false
+export const influxWrite = async (data) => {
+    let start = DateTime.now()
+    let ok = false
+    try {
+        const ret = await axios({
+            method: 'post',
+            url: INFLUXURL_WRITE,
+            data: data,
+            headers: {
+                Authorization: `Token ${INFLUXTOKEN}`,
+                Accept: 'application/json',
+                'Content-Type': 'text/plain; charset=utf-8'
+            },
+            timeout: 10000,
+        })
+        if (ret.status != 204) {
+            logerror(`doWrite2API Status: ${ret.status}`)
+        } else {
+            ok = true
+        }
+    } catch (e) {
+        logerror(`doWrite2API  ${INFLUXURL_WRITE}  ${describeError(e)}`)
+    }
+    statistics['writeInfluxData[sensor_data]Time'] = DateTime.now().diff(start, ['seconds']).toObject().seconds
+    logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
+    return ok
+}
@@ -0,0 +1,15 @@
+import { DateTime} from 'luxon'
+
+const MOCHA_TEST = process.env.MOCHA_TEST || false
+
+export function logit(str) {
+    if(MOCHA_TEST) return
+    let s = `${DateTime.now().toISO()} => ${str}`;
+    console.log(s);
+}
+
+export function logerror(str) {
+    if(MOCHA_TEST) return
+    let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`;
+    console.log(s);
+}
@@ -0,0 +1,312 @@
+/* Interface for MongoDB
+
+Gemeinsam genutzt von readin (Live-API) und readarchive (CSV-Archiv).
+Zusammengefuehrt aus den beiden zuvor getrennten Fassungen; wo sie sich
+unterschieden, gilt:
+  - MONGOAUTH wird als String verglichen. Die alte readarchive-Fassung
+    pruefte nur auf truthy, dadurch schaltete auch MONGOAUTH=false die
+    Authentifizierung ein.
+  - writeDataArray(client, coll, data) nimmt den Collection-Namen direkt.
+    readarchive baut ihn mit dataCollName(styp).
+  - getallProperties liefert das Array selbst (readin-Fassung). Die
+    readarchive-Fassung mit {error, properties} war dort ungenutzt.
+*/
+import { MongoClient } from 'mongodb'
+import { logit, logerror } from './logit.js'
+import { statistics } from './statistics.js'
+import { DateTime } from 'luxon'
+
+let DEVELOP = process.env.DEVELOP || 'false'
+
+let MONGOHOST = process.env.MONGOHOST || 'localhost'
+let MONGOPORT = process.env.MONGOPORT || 27017
+let MONGOAUTH = process.env.MONGOAUTH || 'false'
+let MONGOUSRP = process.env.MONGOUSRP || ''
+let MONGOBASE = process.env.MONGOBASE || 'sensor_data'
+
+let MONGO_URL = 'mongodb://' + MONGOHOST + ':' + MONGOPORT;  	// URL to mongo database
+if (MONGOAUTH == 'true') {
+    MONGO_URL = 'mongodb://' + MONGOUSRP + '@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin';          // URL to mongo database
+}
+
+export const property_coll = 'properties'
+const data_collection = 'sensors'
+
+// Datencollection eines Sensortyps, z.B. dataCollName('noise') -> 'noise_sensors'
+export const dataCollName = (styp) => {
+    return styp + '_' + data_collection
+}
+
+const addandshowstatistics = (client, text, field, start) => {
+    statistics[field] = DateTime.now().diff(start, ['seconds']).toObject().seconds
+    logit(`Write ${text} to mongoDB: Time: ${statistics[field]} sec.`)
+}
+
+export const connectMongo = async () => {
+    try {
+        if(DEVELOP === 'true')  {
+            logit(`Try to connect to ${MONGO_URL}`)
+        } else {
+            logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`)
+        }
+        let client = await MongoClient.connect(MONGO_URL)
+        if ( DEVELOP === 'true') {
+            logit(`Mongodbase connected to ${MONGO_URL}`)
+        } else {
+            logit('Mongodbase connected')
+        }
+        return client
+    }
+    catch (error) {
+        throw (error)
+    }
+}
+
+export const closeMongo = async (client) => {
+    try {
+        await client.close()
+    }
+    catch(error){
+        throw(error)
+    }
+}
+
+/* ***************************************************
+// READ routines
+******************************************************/
+
+// Read properties from the database - oeffnet und schliesst selbst
+export const readProperties = async (query, limit = 0) => {
+    let ret = {err: null, properties: null}
+    let client = await connectMongo()
+    try {
+        if ("sid" in query) {                               // if sid is given, read property for sid
+            ret.properties = await client.db(MONGOBASE).collection(property_coll).findOne({_id: query.sid})
+        } else {                                            // otherwise read props corresponding to query
+            ret.properties = await client.db(MONGOBASE).collection(property_coll).find(query).limit(limit).toArray()
+        }
+    } catch (e) {
+        ret.err = e
+    }
+    finally {
+        client.close()
+    }
+    return ret
+}
+
+export const getallProperties = async (client) => {
+    return await client.db(MONGOBASE).collection(property_coll)
+        .find().sort({ _id: 1 }).toArray()
+}
+
+export const checkOneproperty = async (client, sid) => {
+    return await client.db(MONGOBASE).collection(property_coll)
+        .findOne({ _id: sid })
+}
+
+export const getOneproperty = async (client, sid) => {
+    let ret = {error: false, errortext: '', property: null}
+    try {
+        ret.property = await client.db(MONGOBASE).collection(property_coll)
+            .findOne({_id: sid})
+    } catch (e) {
+        ret =  {error: true, errortext: e}
+    }
+    return ret
+}
+
+export const getOneSensorOneday = async (client, sid, day, styp) => {
+    let ret = {error: false, errortext: '', date: []}
+    // muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC),
+    // sonst passt das Suchfenster nicht auf die Daten des Tages
+    let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' })
+    let start = d.startOf('day').toJSDate()
+    let end = d.startOf('day').plus({day:1}).toJSDate()
+    try {
+        let erg = await client.db(MONGOBASE).collection(dataCollName(styp))
+            .find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray()
+        ret.data = erg
+    } catch(e) {
+        ret.error = true
+        ret.errortext = e
+    }
+    return ret
+}
+
+export const getLocationIDs = async (client, stype) => {
+    let ret = {error: false, errortext: '', locations: []}
+    try {
+        ret.locations = await client.db(MONGOBASE).collection(property_coll)
+            .distinct('location.0.id', {type: stype})
+    } catch (e) {
+        ret.error = true
+        ret.errortext = e
+    }
+    return ret
+}
+
+export const getTHPSensors = async (client, lid) => {
+    let ret = {error: false, errortext: '', sensors: []}
+    try {
+        ret.sensors = await client.db(MONGOBASE).collection(property_coll)
+            .findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}})
+    } catch (e) {
+        ret.error = true
+        ret.errortext = e
+    }
+    return ret
+}
+
+/* ***************************************************
+// WRITE routines
+******************************************************/
+
+export const writeOneproperty = async (client, prop) => {
+    try {
+        let result = await client.db(MONGOBASE).collection(property_coll)
+            .insertOne(prop)
+    } catch (e) {
+        if (e.code == 11000) {
+            return false
+        } else {
+            throw (e)
+        }
+    }
+    return true
+}
+
+export const writeProperties = async (client, props) => {
+    let result
+    let startAll = DateTime.now();
+    let start
+    let coll = client.db(MONGOBASE).collection(property_coll)
+    if (props.new.length !== 0) {
+        start = DateTime.now();
+        try {
+            result = await coll.insertMany(props.new)
+        } catch (e) {
+            logerror(`Write properties new   ${e}`)
+        }
+        logit(`Write ${props.new.length} properties NEW to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
+    }
+    if (props.loc.length !== 0) {
+        start = DateTime.now()
+        try {
+            for (let item of props.loc) {
+                result = await coll.updateOne({ _id: item._id }, {
+                    $set: { location_id: item.location_id },
+                    $push: { location: { $each: [item.location[0]], $position: 0 } }
+                })
+            }
+        } catch (e) {
+            logerror(`Write properties location   ${e}`)
+        }
+        logit(`Write ${props.loc.length} properties LOC to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
+    }
+    if (props.sname.length !== 0) {
+        start = DateTime.now()
+        try {
+            for (let item of props.sname) {
+                result = await coll.updateOne({ _id: item._id }, {
+                    $push: { name: { $each: [item.name[0]], $position: 0 } }
+                })
+            }
+        } catch (e) {
+            logerror(`Write properties samename   ${e}`)
+        }
+        logit(`Write ${props.sname.length} properties NAME to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
+    }
+    addandshowstatistics(client, 'properties', 'writePropsTime', startAll)
+}
+
+export const writeOneSensor = async (client, data, styp) => {
+    let ret = {error: false, errortext: '', inserted: 1}
+    try {
+        await client.db(MONGOBASE).collection(dataCollName(styp))
+            .insertOne(data)
+    } catch (e) {
+        ret.error = true
+        ret.errortext = e
+        ret.inserted = 0
+    }
+    return ret
+}
+
+export const writeDataArray = async (client, coll, data) => {
+    let result
+    let start = DateTime.now();
+    try {
+        result = await client.db(MONGOBASE).collection(coll)
+            .insertMany(data, { ordered: false })
+    } catch (e) {
+        if (e.code !== 11000) {
+            console.error(e)
+        }
+    }
+    addandshowstatistics(client, `${data.length} entries for ${coll}`, `writeMongoData[${coll}]Time`, start)
+}
+
+export const writeStatistic = async (client, stat) => {
+    let result
+    let start = DateTime.now();
+    let entry = { timestamp: new Date(), ...stat }
+    try {
+        result = await client.db(MONGOBASE).collection("statistics")
+            .insertOne(entry)
+    } catch (e) {
+        console.error(e)
+    }
+    addandshowstatistics(client, `statistics`, `writeStatisticTime`, start)
+}
+
+export const bulkWrite = async (client, coll, data) => {
+    let start = DateTime.now()
+    let result
+    try {
+        result = await client.db(MONGOBASE).collection(coll)
+            .bulkWrite(data, { ordered: false })
+    } catch (e) {
+        console.error(e)
+    }
+    addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start)
+    return result
+}
+
+export const bulkUpdateMapdata = async (client, data) => {
+    let start = DateTime.now()
+    let result
+    try {
+        result = await client.db(MONGOBASE).collection("mapdata")
+            .bulkWrite(data, { ordered: false })
+    } catch (e) {
+        console.error(e)
+    }
+    logit(`Write MapData: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
+    return result
+}
+
+/* ***************************************************
+// Wartung
+******************************************************/
+
+export const dropColl = async (client, coll) => {
+    let start = DateTime.now()
+    let result
+    try {
+        result = await client.db(MONGOBASE).collection(coll).drop()
+    } catch (e) {
+        console.error(e)
+    }
+    logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
+}
+
+export const createIndex = async (client, coll) => {
+    let result
+    let start = DateTime.now()
+    try {
+        result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" })
+    } catch (e) {
+        console.error(e)
+    }
+    logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
+}
@@ -0,0 +1,7 @@
+/* Gemeinsames Sammelobjekt fuer Laufzeit-Statistiken.
+
+Lag frueher in readin/readdata.js. Liegt hier, damit auch die gemeinsam
+genutzten Module darauf schreiben koennen, ohne readin kennen zu muessen.
+*/
+
+export const statistics = {}