first
This commit is contained in:
240
parse.js
Normal file
240
parse.js
Normal file
@@ -0,0 +1,240 @@
|
||||
// import logit from './logit.js'
|
||||
import { DateTime } from 'luxon'
|
||||
import * as mongo from './mongo.js'
|
||||
import { statistics } from'./readdata.js'
|
||||
import { logit, logerror } from './logit.js'
|
||||
|
||||
let actualProps = []
|
||||
let newProps = []
|
||||
|
||||
// Check lat/lon and convert to float
|
||||
function checkLatLon(w) {
|
||||
let loc = 0.0;
|
||||
if (!((w === undefined) || (w == null) || (w == ''))) {
|
||||
try {
|
||||
loc = parseFloat(w)
|
||||
} catch (e) {
|
||||
logerror(`Math error with lat/lon, ${e}`)
|
||||
}
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
// Check, if altitude is there. If so, use it, else use 0
|
||||
function checkAltitude(alt) {
|
||||
let altitude = 0;
|
||||
if(!((alt === undefined) || (alt === ''))) {
|
||||
try {
|
||||
altitude = Math.floor(parseFloat(alt))
|
||||
} catch (e) {
|
||||
logerror(`Math error with altitude, ${e}`)
|
||||
}
|
||||
}
|
||||
return altitude
|
||||
}
|
||||
|
||||
// binary search fir sensor id in property array
|
||||
const binarySearch = (arr, element, x) => {
|
||||
let start = 0, end = arr.length - 1
|
||||
// Iterate while start not meets end
|
||||
while (start <= end) {
|
||||
// Find the mid index
|
||||
let mid = Math.floor((start + end) / 2)
|
||||
// If element is present at mid, return True
|
||||
if (arr[mid][element] === x) return mid
|
||||
// Else look in left or right half accordingly
|
||||
else if (arr[mid][element] < x)
|
||||
start = mid + 1
|
||||
else
|
||||
end = mid - 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
|
||||
const checkProperties = (item, mapvalues, typ, dt) => {
|
||||
let now = DateTime.utc().toJSDate()
|
||||
let entry
|
||||
// read entry from actualprops
|
||||
let idx = binarySearch(actualProps, '_id', item.sensor.id)
|
||||
if (idx === -1) { // not in properties => new sensor
|
||||
entry = buildNewEntry(item, typ, dt, now) // so build a new entry
|
||||
} else {
|
||||
entry = actualProps[idx] // get actual properties
|
||||
// check for change of location
|
||||
if (entry.location[0].id !== item.location.id) { // have got a new location
|
||||
if (newProps.findIndex((obj) => {
|
||||
return (obj.replaceOne.replacement.location[0].id === item.location.id)
|
||||
}) === -1) {
|
||||
const newloc = {
|
||||
loc: {
|
||||
type: "Point",
|
||||
coordinates: [
|
||||
checkLatLon(item.location.longitude),
|
||||
checkLatLon(item.location.latitude)
|
||||
]
|
||||
},
|
||||
id: item.location.id,
|
||||
altitude: checkAltitude(item.location.altitude),
|
||||
since: now,
|
||||
exact_loc: item.location.exact_location,
|
||||
indoor: item.location.indoor,
|
||||
country: item.location.country
|
||||
}
|
||||
entry.location.splice(0, 0, newloc) // insert new location at pos 0 in location array
|
||||
}
|
||||
} else { // same location check for change in indoor or exact_location
|
||||
if (entry.location[0].indoor !== item.location.indoor) {
|
||||
entry.location[0].indoor = item.location.indoor // update indoor
|
||||
}
|
||||
if (entry.location[0].exact_loc !== item.location.exact_location) {
|
||||
entry.location[0].exact_loc = item.location.exact_location // update exact_location
|
||||
}
|
||||
if (entry.location[0].country === '') {
|
||||
entry.location[0].country = item.location.country // update country
|
||||
}
|
||||
}
|
||||
// Check für new name
|
||||
if (entry.name[0].name !== item.sensor.sensor_type.name) { // have got a new name
|
||||
if (newProps.findIndex((obj) => {
|
||||
return (obj.replaceOne.replacement.name[0].name === item.sensor.sensor_type.name)
|
||||
}) === -1) {
|
||||
let newname = {
|
||||
name: item.sensor.sensor_type.name,
|
||||
since: now
|
||||
}
|
||||
entry.name.splice(0, 0, newname)
|
||||
}
|
||||
}
|
||||
}
|
||||
// set new mapvalues
|
||||
entry.values = mapvalues
|
||||
delete entry.location_id
|
||||
delete entry.last_seen
|
||||
delete entry.since
|
||||
// push this entry to the new proerties array
|
||||
newProps.push({replaceOne: { filter: {_id: entry._id}, replacement: entry, upsert: true}})
|
||||
}
|
||||
|
||||
|
||||
const buildNewEntry = (item, typ, dt, now) => {
|
||||
return {
|
||||
_id: item.sensor.id,
|
||||
type: typ,
|
||||
name: [{
|
||||
name: item.sensor.sensor_type.name,
|
||||
since: now,
|
||||
}],
|
||||
location: [{
|
||||
loc: {
|
||||
type: "Point",
|
||||
coordinates: [
|
||||
checkLatLon(item.location.longitude),
|
||||
checkLatLon(item.location.latitude)
|
||||
]
|
||||
},
|
||||
id: item.location.id,
|
||||
altitude: checkAltitude(item.location.altitude),
|
||||
since: now,
|
||||
exact_loc: item.location.exact_location,
|
||||
indoor: item.location.indoor,
|
||||
country: item.location.country
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
const types = {
|
||||
P1: 'pm', P2: 'pm', P0: 'pm',
|
||||
temperature: 'thp', humidity: 'thp', pressure: 'thp',
|
||||
noise_LAeq: 'noise', noise_LA_max: 'noise',
|
||||
counts_per_minute: 'radioactivity',
|
||||
lat: 'gps'
|
||||
};
|
||||
|
||||
|
||||
function getType(typ) {
|
||||
if(typ in types) {
|
||||
return types[typ];
|
||||
} else {
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
export const constructDBaseEntries = async (client, body, args) => {
|
||||
logit(`Number of entries: ${body.length}`)
|
||||
logit('Parsing ...')
|
||||
let start = DateTime.now()
|
||||
let allLines = {
|
||||
pm: [], thp: [], noise: [], radioactivity: [], gps: [], unknown: []
|
||||
}
|
||||
let datalines = ''
|
||||
actualProps = await mongo.getallProperties(client)
|
||||
try {
|
||||
for (let item of body) { // check all entries
|
||||
const dt = item.timestamp.split(' ')
|
||||
const datetime = new Date(dt[0] + 'T' + dt[1] + 'Z') // extract date of entry as utc
|
||||
let values = {}
|
||||
let ival = '' // fetch values
|
||||
let typ = 'unknown'
|
||||
let mapvalue = {}
|
||||
for (let v of item.sensordatavalues) { // for all values
|
||||
let vtyp = v.value_type; // extract value type
|
||||
if (typ === 'unknown') { // extract measurement type
|
||||
typ = getType(vtyp)
|
||||
}
|
||||
let val = v.value; // and value
|
||||
let x
|
||||
try {
|
||||
x = parseFloat(val); // convert value to float
|
||||
if (Number.isNaN(x)) {
|
||||
x = -9999.9 // default if value is invalid or unknown
|
||||
}
|
||||
} catch (err) {
|
||||
console.log('Math parse float error on value');
|
||||
x = -9999.9 // default if value is invalid or unknown
|
||||
}
|
||||
values[vtyp] = x
|
||||
ival += `${vtyp}=${x},`
|
||||
// if noise sensor precalculate pow10
|
||||
if (vtyp == 'noise_LAeq') {
|
||||
let e10 = Math.pow(10, x / 10)
|
||||
values.E10tel_eq = e10
|
||||
ival += `E10tel_eq=${e10},`
|
||||
mapvalue.E10tel_eq = e10
|
||||
}
|
||||
mapvalue[vtyp] = x
|
||||
}
|
||||
let store = true
|
||||
if(args.typ) {
|
||||
if(!args.typ.includes(typ)) {
|
||||
store = false
|
||||
}
|
||||
}
|
||||
if (store) {
|
||||
allLines[typ].push({sensorid: item.sensor.id, datetime: datetime, values: values})
|
||||
datalines += `${typ},sid=${item.sensor.id} ${ival.slice(0,-1)} ${datetime.getTime()}\n`
|
||||
mapvalue.timestamp = datetime
|
||||
checkProperties(item, mapvalue, typ, datetime); // check if new or new location or new sensor
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logerror(`constructDBaseEntries ${e}`);
|
||||
}
|
||||
// sort allLines on sensorID
|
||||
|
||||
for (const [k,v] of Object.entries(allLines)) {
|
||||
allLines[k].sort((a, b) => {
|
||||
if (a.sensorid < b.sensorid) {
|
||||
return -1
|
||||
} else if (a.sensorid > b.sensorid) {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
}
|
||||
statistics.parseTime = DateTime.now().diff(start, ['seconds']).toObject().seconds
|
||||
logit(`Parse time: ${statistics.parseTime} sec`)
|
||||
return [newProps, allLines, datalines]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user