// import logit from './logit.js' import { DateTime } from 'luxon' import * as mongo from './mongo.js' import { statistics } from'./readdata.js' import { logit, logerror } from './logit.js' let actualProps = [] let newProps = [] // Check lat/lon and convert to float function checkLatLon(w) { let loc = 0.0; if (!((w === undefined) || (w == null) || (w == ''))) { try { loc = parseFloat(w) } catch (e) { logerror(`Math error with lat/lon, ${e}`) } } return loc } // Check, if altitude is there. If so, use it, else use 0 function checkAltitude(alt) { let altitude = 0; if(!((alt === undefined) || (alt === ''))) { try { altitude = Math.floor(parseFloat(alt)) } catch (e) { logerror(`Math error with altitude, ${e}`) } } return altitude } // binary search fir sensor id in property array const binarySearch = (arr, element, x) => { let start = 0, end = arr.length - 1 // Iterate while start not meets end while (start <= end) { // Find the mid index let mid = Math.floor((start + end) / 2) // If element is present at mid, return True if (arr[mid][element] === x) return mid // Else look in left or right half accordingly else if (arr[mid][element] < x) start = mid + 1 else end = mid - 1 } return -1 } const checkProperties = (item, mapvalues, typ, dt) => { let now = DateTime.utc().toJSDate() let entry // read entry from actualprops let idx = binarySearch(actualProps, '_id', item.sensor.id) if (idx === -1) { // not in properties => new sensor entry = buildNewEntry(item, typ, dt, now) // so build a new entry } else { entry = actualProps[idx] // get actual properties // check for change of location if (entry.location[0].id !== item.location.id) { // have got a new location if (newProps.findIndex((obj) => { return (obj.replaceOne.replacement.location[0].id === item.location.id) }) === -1) { const newloc = { loc: { type: "Point", coordinates: [ checkLatLon(item.location.longitude), checkLatLon(item.location.latitude) ] }, id: item.location.id, altitude: checkAltitude(item.location.altitude), since: now, exact_loc: item.location.exact_location, indoor: item.location.indoor, country: item.location.country } entry.location.splice(0, 0, newloc) // insert new location at pos 0 in location array } } else { // same location check for change in indoor or exact_location if (entry.location[0].indoor !== item.location.indoor) { entry.location[0].indoor = item.location.indoor // update indoor } if (entry.location[0].exact_loc !== item.location.exact_location) { entry.location[0].exact_loc = item.location.exact_location // update exact_location } if (entry.location[0].country === '') { entry.location[0].country = item.location.country // update country } } // Check für new name if (entry.name[0].name !== item.sensor.sensor_type.name) { // have got a new name if (newProps.findIndex((obj) => { return (obj.replaceOne.replacement.name[0].name === item.sensor.sensor_type.name) }) === -1) { let newname = { name: item.sensor.sensor_type.name, since: now } entry.name.splice(0, 0, newname) } } } // set new mapvalues entry.values = mapvalues delete entry.location_id delete entry.last_seen delete entry.since // push this entry to the new proerties array newProps.push({replaceOne: { filter: {_id: entry._id}, replacement: entry, upsert: true}}) } const buildNewEntry = (item, typ, dt, now) => { return { _id: item.sensor.id, type: typ, name: [{ name: item.sensor.sensor_type.name, since: now, }], location: [{ loc: { type: "Point", coordinates: [ checkLatLon(item.location.longitude), checkLatLon(item.location.latitude) ] }, id: item.location.id, altitude: checkAltitude(item.location.altitude), since: now, exact_loc: item.location.exact_location, indoor: item.location.indoor, country: item.location.country }] } } const types = { P1: 'pm', P2: 'pm', P0: 'pm', temperature: 'thp', humidity: 'thp', pressure: 'thp', noise_LAeq: 'noise', noise_LA_max: 'noise', counts_per_minute: 'radioactivity', lat: 'gps' }; function getType(typ) { if(typ in types) { return types[typ]; } else { return 'unknown' } } export const constructDBaseEntries = async (client, body, args) => { logit(`Number of entries: ${body.length}`) logit('Parsing ...') let start = DateTime.now() let allLines = { pm: [], thp: [], noise: [], radioactivity: [], gps: [], unknown: [] } let datalines = '' actualProps = await mongo.getallProperties(client) try { for (let item of body) { // check all entries const dt = item.timestamp.split(' ') const datetime = new Date(dt[0] + 'T' + dt[1] + 'Z') // extract date of entry as utc let values = {} let ival = '' // fetch values let typ = 'unknown' let mapvalue = {} for (let v of item.sensordatavalues) { // for all values let vtyp = v.value_type; // extract value type if (typ === 'unknown') { // extract measurement type typ = getType(vtyp) } let val = v.value; // and value let x try { x = parseFloat(val); // convert value to float if (Number.isNaN(x)) { x = -9999.9 // default if value is invalid or unknown } } catch (err) { console.log('Math parse float error on value'); x = -9999.9 // default if value is invalid or unknown } values[vtyp] = x ival += `${vtyp}=${x},` // if noise sensor precalculate pow10 if (vtyp == 'noise_LAeq') { let e10 = Math.pow(10, x / 10) values.E10tel_eq = e10 ival += `E10tel_eq=${e10},` mapvalue.E10tel_eq = e10 } mapvalue[vtyp] = x } let store = true if(args.typ) { if(!args.typ.includes(typ)) { store = false } } if (store) { allLines[typ].push({sensorid: item.sensor.id, datetime: datetime, values: values}) datalines += `${typ},sid=${item.sensor.id} ${ival.slice(0,-1)} ${datetime.getTime()}\n` mapvalue.timestamp = datetime checkProperties(item, mapvalue, typ, datetime); // check if new or new location or new sensor } } } catch (e) { logerror(`constructDBaseEntries ${e}`); } // sort allLines on sensorID for (const [k,v] of Object.entries(allLines)) { allLines[k].sort((a, b) => { if (a.sensorid < b.sensorid) { return -1 } else if (a.sensorid > b.sensorid) { return 1 } else { return 0 } }) } statistics.parseTime = DateTime.now().diff(start, ['seconds']).toObject().seconds logit(`Parse time: ${statistics.parseTime} sec`) return [newProps, allLines, datalines] }