readarchive: Import aus dem CSV-Archiv dazu

Gegenstueck zu readin/: waehrend readin alle 5 Minuten die Live-API abfragt, liest readarchive rueckwirkend die Tagesdateien von archive.sensor.community in dieselben Datenbanken ein. Ohne History uebernommen (das Projekt lag bisher in einem eigenen Repository unter Sensors/Laerm/laerm_readfromcsv_to_database). Stand entspricht dort e33a7b7: - Zeitstempel werden in UTC gespeichert, unabhaengig von der Zeitzone der Maschine. readin/parse.js macht das seit jeher richtig, readFromcsv.js lag zwei Stunden daneben. - Fehlgeschlagene Influx-Writes werden gemeldet und setzen den Exit-Code, statt still verloren zu gehen. - Der Container nimmt Parameter entgegen (ENTRYPOINT in Exec-Form). Hinweis: mongo.js, influx_post.js und logit.js gibt es auch unter readin/, mit abweichendem Stand. Zusammenfuehren waere der naechste Schritt. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 06:36:01 +00:00
parent d6e6cb3f76
commit 7c6692c040
15 changed files with 1593 additions and 0 deletions
@@ -0,0 +1,134 @@
+import { DateTime } from 'luxon'
+import { logit, logerror } from './logit.js'
+import { returnOnError } from './reporterror.js'
+import * as mongo from '../mongo.js'
+
+// Check lat/lon and convert to float
+function checkLatLon(w) {
+    let loc = 0.0;
+    if (!((w === undefined) || (w == null) || (w == ''))) {
+        try {
+            loc = parseFloat(w)
+        } catch (e) {
+            logerror(`Math error with lat/lon, ${e}`)
+        }
+    }
+    return loc
+}
+
+
+const buildNewEntry = (item, cdate, indoor, name, typ) => {
+    return {
+        _id:            parseInt(item.sensor_id),
+        type:           typ,
+        name:           [{
+            name: name,
+            since: cdate,
+        }],
+        location:       [{
+            loc: {
+                type:   "Point",
+                coordinates: [
+                    checkLatLon(item.lon),
+                    checkLatLon(item.lat)
+                ]
+            },
+            id:         parseInt(item.location),
+            altitude:   0,
+            since:      cdate,
+            exact_loc:  0,
+            indoor:     indoor,
+            country:    ''
+        }]
+    }
+}
+
+// Check for new properties
+export const checkProperties = async (client, item, indoor, name, typ, dt, newProps) => {
+    if(name === "laerm") {
+        name = 'DNMS (Laerm)'
+    }
+    let ret
+    try {
+        ret = await mongo.getOneproperty(client, parseInt(item.sensor_id) ) // read properties for
+        if(ret.error) {
+            logerror(`Error reading properties: ${ret.errortext}`)
+            return null
+        }
+    } catch (e) {
+        logerror(`Error reading properties: ${e}`)
+        return null
+    }
+    let changed = false
+    let entry = ret.property
+    let cdate = DateTime.fromISO(dt + 'T00:00:00', { zone: 'utc' }).toJSDate()
+    // read entry from actualprops
+    if (entry === null) {                                       // not in properties => new sensor
+        entry = buildNewEntry(item, cdate, indoor, name, typ)      // so build a new entry
+        changed = true
+    } else {
+        // check for change of collection
+        let ok = false
+        for(let i = 0; i < entry.location.length; i++) {
+            if(entry.location[i].id === parseInt(item.location)) {
+                ok = true                                       // same location found
+            }
+        }
+        if( !ok) {
+            // new location found:
+            if (newProps.findIndex((obj) => {
+                    return (obj.updateOne.update.$set.location[0].id === item.location)
+                }) === -1) {                                        // not in newProps array
+                    const newloc = {
+                        loc: {
+                            type: "Point",
+                            coordinates: [
+                                checkLatLon(item.lon),
+                                checkLatLon(item.lat)
+                            ]
+                        },
+                        id: parseInt(item.location),
+                        altitude: 0,
+                        since: cdate,
+                        exact_loc: 0,
+                        indoor: indoor
+                    }
+                    entry.location.push(newloc)             // add new lolcation to array
+                    changed = true
+                    logit(`New location ${item.location} for sensor ${item.sensor_id}`)
+            }
+        } else {
+                                           // same location, check the date
+            if (entry.location[0].since > cdate) {            // new date is older than old date
+                entry.location[0].since = cdate               // so set new date
+                changed = true
+            }
+        }
+        // Check für new name
+        if (entry.name[0].name.toUpperCase() !== name.toUpperCase()) {                  // have got a new name
+            if (newProps.findIndex(
+                (obj) => {
+                return (obj.updateOne.update.$set.name[0].name === name)
+            }) === -1) {
+                let newname = {
+                    name: name,
+                    since: cdate
+                }
+                entry.name.push(newname)
+                changed = true
+                logit(`New name ${name} for sensor ${item.sensor_id}`)
+            }
+        } else {                                            // same name, chack the date
+            if (entry.name[0].since > cdate) {                // new date is older than old date
+                entry.name[0].since = cdate                   // so set new date
+                changed = true
+                logit(`New date for sensor ${item.sensor_id}`)
+            }
+        }
+    }
+    // push this entry to the new properties array if somethind was changed
+    if(changed) {
+        delete entry.values                                   // delete values array
+        newProps.push({updateOne: {filter: {_id: entry._id}, update: {$set: entry}, upsert: true}})
+    }
+}