readarchive: Import aus dem CSV-Archiv dazu
Gegenstueck zu readin/: waehrend readin alle 5 Minuten die Live-API abfragt, liest readarchive rueckwirkend die Tagesdateien von archive.sensor.community in dieselben Datenbanken ein. Ohne History uebernommen (das Projekt lag bisher in einem eigenen Repository unter Sensors/Laerm/laerm_readfromcsv_to_database). Stand entspricht dort e33a7b7: - Zeitstempel werden in UTC gespeichert, unabhaengig von der Zeitzone der Maschine. readin/parse.js macht das seit jeher richtig, readFromcsv.js lag zwei Stunden daneben. - Fehlgeschlagene Influx-Writes werden gemeldet und setzen den Exit-Code, statt still verloren zu gehen. - Der Container nimmt Parameter entgegen (ENTRYPOINT in Exec-Form). Hinweis: mongo.js, influx_post.js und logit.js gibt es auch unter readin/, mit abweichendem Stand. Zusammenfuehren waere der naechste Schritt. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
/* Interface for MongoDB
|
||||
*/
|
||||
import { MongoClient } from 'mongodb'
|
||||
import { logit, logerror } from './utilities/logit.js'
|
||||
import { DateTime } from 'luxon'
|
||||
import {returnOnError} from "./utilities/reporterror.js";
|
||||
|
||||
const DEVELOP = process.env.DEVELOP || false;
|
||||
|
||||
const MONGOHOST = process.env.MONGOHOST || 'localhost';
|
||||
const MONGOPORT = process.env.MONGOPORT || 27017;
|
||||
const MONGOAUTH = process.env.MONGOAUTH || false;
|
||||
const MONGOUSRP = process.env.MONGOUSRP || "";
|
||||
const MONGOBASE = process.env.MONGOBASE || 'sensor_data';
|
||||
|
||||
const MONGO_URL = MONGOAUTH ? 'mongodb://'+MONGOUSRP+'@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin' : 'mongodb://'+MONGOHOST+':'+MONGOPORT; // URL to mongo database
|
||||
if (DEVELOP) {
|
||||
console.log(`MongoURL = "${MONGO_URL}" and Database = ${MONGOBASE}`);
|
||||
}
|
||||
const statistics = {}
|
||||
|
||||
export const properties_collection = 'properties'
|
||||
const data_collection = 'sensors'
|
||||
|
||||
export const connectMongo = async () => {
|
||||
try {
|
||||
if(DEVELOP === 'true') {
|
||||
logit(`Try to connect to ${MONGO_URL}`)
|
||||
} else {
|
||||
logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`)
|
||||
}
|
||||
let client = await MongoClient.connect(MONGO_URL)
|
||||
if ( DEVELOP === 'true') {
|
||||
logit(`Mongodbase connected to ${MONGO_URL}`)
|
||||
} else {
|
||||
logit('Mongodbase connected')
|
||||
}
|
||||
return client
|
||||
}
|
||||
catch (error) {
|
||||
throw (error)
|
||||
}
|
||||
}
|
||||
|
||||
export const closeMongo = async (client) => {
|
||||
try {
|
||||
await client.close()
|
||||
}
|
||||
catch(error){
|
||||
throw(error)
|
||||
}
|
||||
}
|
||||
|
||||
const listDatabases = async (client) => {
|
||||
let databasesList = await client.db().admin().listDatabases();
|
||||
|
||||
console.log("Databases:");
|
||||
databasesList.databases.forEach(db => console.log(` - ${db.name}`));
|
||||
}
|
||||
|
||||
/* ***************************************************
|
||||
// READ routines
|
||||
******************************************************/
|
||||
|
||||
// Read properties from the database
|
||||
export const readProperties = async (query, limit = 0) => {
|
||||
let ret = {err: null, properties: null}
|
||||
let client = await connectMongo()
|
||||
try {
|
||||
if ("sid" in query) { // if sid is given, read property for sid
|
||||
ret.properties = await client.db(MONGOBASE).collection(properties_collection).findOne({_id: query.sid})
|
||||
} else { // otherwise read props corresponding to query
|
||||
ret.properties = await client.db(MONGOBASE).collection(properties_collection).find(query).limit(limit).toArray()
|
||||
}
|
||||
} catch (e) {
|
||||
ret.err = e
|
||||
}
|
||||
finally {
|
||||
client.close()
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
|
||||
export const getallProperties = async (client) => {
|
||||
let ret = {error: false, errortext: '', properties: []}
|
||||
try
|
||||
{
|
||||
ret.properties = await client.db(MONGOBASE).collection(properties_collection)
|
||||
.find().sort({_id: 1}).toArray()
|
||||
}
|
||||
catch(e) {
|
||||
ret = {error: true, errortext: e}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
|
||||
export const getOneproperty = async (client, sid) => {
|
||||
let ret = {error: false, errortext: '', property: null}
|
||||
try {
|
||||
ret.property = await client.db(MONGOBASE).collection(properties_collection)
|
||||
.findOne({_id: sid})
|
||||
} catch (e) {
|
||||
ret = {error: true, errortext: e}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
const getCollName = (coll, st) => {
|
||||
return st + '_' + coll
|
||||
}
|
||||
|
||||
export const getOneSensorOneday = async (client, sid, day, styp) => {
|
||||
let ret = {error: false, errortext: '', date: []}
|
||||
// muss dieselbe Zone benutzen wie die gespeicherten Zeitstempel (UTC),
|
||||
// sonst passt das Suchfenster nicht auf die Daten des Tages
|
||||
let d = DateTime.fromFormat(day, "yyyy-LL-dd", { zone: 'utc' })
|
||||
let start = d.startOf('day').toJSDate()
|
||||
let end = d.startOf('day').plus({day:1}).toJSDate()
|
||||
try {
|
||||
let erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
|
||||
.find({sensorid: sid, datetime: {$gte: start, $lt: end}},{sort: {datetime: 1}}).toArray()
|
||||
ret.data = erg
|
||||
} catch(e) {
|
||||
ret.error = true
|
||||
ret.errortext = e
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
export const writeOneSensor = async (client, data, styp) => {
|
||||
let ret = {error: false, errortext: '', inserted: 1}
|
||||
let erg
|
||||
try {
|
||||
erg = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
|
||||
.insertOne(data)
|
||||
} catch (e) {
|
||||
ret.error = true
|
||||
ret.errortext = e
|
||||
ret.inserted = 0
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
export const writeDataArray = async (client, data, styp) => {
|
||||
let result
|
||||
let start = DateTime.now();
|
||||
try {
|
||||
result = await client.db(MONGOBASE).collection(getCollName(data_collection, styp))
|
||||
.insertMany(data, {ordered: false})
|
||||
} catch (e) {
|
||||
if(e.code !== 11000) {
|
||||
console.error(e)
|
||||
}
|
||||
}
|
||||
let statname = `writeData${getCollName(data_collection, styp)}Time`
|
||||
statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
|
||||
logit(`Write Data for ${getCollName(data_collection, styp)} to mongoDB: Time: ${statistics[statname]} sec.`)
|
||||
}
|
||||
|
||||
export const bulkWrite = async (client, coll, data) => {
|
||||
let start = DateTime.now()
|
||||
let result
|
||||
try {
|
||||
result = await client.db(MONGOBASE).collection(coll)
|
||||
.bulkWrite(data, {ordered: false})
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
logit(`Write Properties: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
|
||||
return result
|
||||
}
|
||||
|
||||
export const getLocationIDs = async (client, stype) => {
|
||||
let ret = {error: false, errortext: '', locations: []}
|
||||
try {
|
||||
ret.locations = await client.db(MONGOBASE).collection(properties_collection)
|
||||
.distinct('location.0.id', {type: stype})
|
||||
} catch (e) {
|
||||
ret.error = true
|
||||
ret.errortext = e
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
export const getTHPSensors = async (client, lid) => {
|
||||
let ret = {error: false, errortext: '', sensors: []}
|
||||
try {
|
||||
ret.sensors = await client.db(MONGOBASE).collection(properties_collection)
|
||||
.findOne({type: 'thp', 'location.0.id': lid}, { projection: {_id: 1}})
|
||||
} catch (e) {
|
||||
ret.error = true
|
||||
ret.errortext = e
|
||||
}
|
||||
return ret
|
||||
}
|
||||
Reference in New Issue
Block a user