sensorapi/sensorspecials/noise.js

// Data preparation for fetching noise data
// rxf 2023-03-05

import {returnOnError} from "../utilities/reporterror.js";
import { getActData, getAvgData, getLongAvg, fetchFromInflux, calcRange} from "../actions/getsensorData.js"
import checkParams from "../utilities/checkparams.js";
import * as ERR from "../utilities/errortexts.js"
import {DateTime} from 'luxon'

const setoptionfromtable = (opt,tabval) => {
    let ret = opt
    if ((opt === null) || (opt === '') || (opt < tabval)){
         ret = tabval
    }
    return ret
}


export const getNoiseData = async (params, possibles, props) => {
    let ret = {err: null}

    let {opts, err} = checkParams(params, {
        mandatory:[
            {name:'sensorid', type: 'int'},
        ],
        optional: possibles
    })
    // To be compatible with old API:
    if (opts.out === 'csv') {
        opts.csv = true
    }
    if (err) {
        return returnOnError(ret, err, getNoiseData.name)
    }
    // execute function depending on given 'data'
    for(let x of whatTable) {
        if (x.what === opts.data) {
            opts.span = setoptionfromtable(opts.span, x.span)
            opts.daystart = setoptionfromtable(opts.daystart, x.daystart)
            let {start, stop} = calcRange(opts)             // calc time range
            opts.start = start
            opts.stop = stop
            let erg = await x.func(opts)                    // get the data
            ret = {
                err: erg.err,
                options: {
                    sid: opts.sensorid,
                    indoor: props.location[0].indoor,
                    span: opts.span,
                    start: opts.start.slice(7),
                    data: opts.data,
                    peak: opts.peak,
                    count: erg.values.length,
                },
                values: erg.values,
            }
            if (!x.peak) {
                delete ret.options.peak
            }
            if(ret.values.length === 0) {
                ret.err = ERR.NODATA
            }
            return ret
        }
    }
    return returnOnError(ret, ERR.CMNDUNKOWN, getNoiseData.name)
}


// *********************************************
// getLiveData
//
// Get all actual data from database. Values are stored every 2.5min
//
//  params:
//      db:             Database
//      opt:            different options (see further down)
//
//  return:
//      JSON:
//      { sid: 29212, span: 1, start: "2019-10-23T00:00", count: 381, values: [
//          { datetime: "2019-10-22T22:05:34.000Z", noise_LAeq: 42.22, noise_LA_min: 39.91, noise_LA_max: 45.18, E10tel_eq: 16672.47212551061 },
//          { datetime: "2019-10-22T22:07:59.000Z", noise_LAeq: 53.72, noise_LA_min: 39.97, noise_LA_max: 63.54, E10tel_eq: 235504.9283896009 },
//              .........
//      ]}
//      CSV
//      datetime,LAeq,LAmax,LAmin,"10^(LAeq/10)"
//      2019-10-22T22:05:34.000Z,42.22,45.18,39.91,16672.47212551061
//      2019-10-22T22:07:59.000Z,53.72,63.54,39.97,235504.9283896009
//      2019-10-22T22:15:16.000Z,44.02,48.99,42.14,25234.807724805756
//          ....
//
// *********************************************
const getLiveData = async (opts) => {
    const erg = await getActData(opts)
    return erg
}


// *********************************************
// gethavgData
//
// Get average per hour, default:  5 days
//
//  params:
//      db:             Database
//      opt:            different options (see further down)
//
//  return:
//      JSON:
//      { sid: 29212, span: 5, start: "2019-11-01T23:00:00Z", average: 'hour', peak: 70, count: 120, values: [
//          { datetime: "2019-10-22T23:00:00.000Z", n_AVG: 58.27, peakcount: 3 },
//          { datetime: "2019-10-23T00:00:00.000Z", n_AVG: 45.77, peakcount: 4 },
//          { datetime: "2019-10-23T01:00:00.000Z", n_AVG: 62.34, peakcount: 6 },
//              .........
//      ]}

//      CSV:
//      datetime,n_AVG,peakcount
//      2019-10-22T23:00:00.000Z,58.27,3
//      2019-10-23T00:00:00.000Z,45.77,4
//      2019-10-23T01:00:00.000Z,62.34,6
//          ....
//
// *********************************************
const gethavgData = async (opts, props) => {
    let erg = await getNoiseAVGData(opts)
    if (opts.csv) {
        let csvStr = "datetime,n_AVG,peakcount\n"
        if (!erg.err) {
            for (let item of erg.values) {
                if (item.n_AVG != -1) {
                    csvStr += item.datetime + ',' + item.n_AVG + ',' + item.peakcount + '\n'
                }
            }
        }
        return csvStr
    }
    return {err: erg.err, values: erg.values}
}


// *********************************************
// getdavgData
//
// Get average per  day , default:  30 days
//
//  params:
//      db:             Database
//      opt:            different options (see further down)
//
//  return:
//      JSON:
//      { sid: 29212, span: 30, start: "2019-10-23T00:00", average: 'day', peak: 70, count: 30, values: [
//          { datetime: "2019-10-22T23:00:00.000Z", n_AVG: 58.27, peakcount: 300 },
//          { datetime: "2019-10-23T23:00:00.000Z", n_AVG: 62.34, peakcount: 245 },
//              .........
//      ]}
//
//      CSV:
//      datetime,n_AVG,peakcount
//      2019-10-22T23:00:00.000Z,58.27,300
//      2019-10-23T23:00:00.000Z,62.34,245
//          ....
//
// *********************************************
async function getdavgData(opts) {
    opts.long = true;
    let erg = await getNoiseAVGData(opts);
    let val = [];
    let csvStr = 'datetime,n_AVG,peakcount\n';
    if (!erg.err) {
        for (let i = 0; i < erg.values.length; i += 24) {
            let sum = 0;
            let count = 0;
            let pk = 0;
            let werte = {};
            for (let k = 0; k < 24; k++) {
                const item = erg.values[i + k]
                if ((item != null) && (item.n_sum != -1)) {
                    sum += item.n_sum;
                    count += item.count;
                    pk += item.peakcount;
                    if (werte.datetime === undefined) {
                        let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                        werte.datetime = dt.startOf('day').toISO()
                    }
                }
            }
            if (count === 0) {
                werte.n_AVG = -1
            } else {
                werte.n_AVG = 10 * Math.log10(sum / count);
            }
            werte.peakcount = pk;
            if (opts.csv) {
                csvStr += werte.datetime + ',' + werte.n_AVG + ',' + werte.peakcount + '\n'
            } else {
                val.push(werte);
            }
        }
    }
    if (opts.csv) {
        return csvStr;
    }
    return {err: erg.err, values: val}
}


// *********************************************
// getdaynightData
//
// Get average for day (6h00 - 22h00) and night (22h00 - 6h00)  separated
// Use the hour average calculation, which brings the sum and the count for every hour
// then add these values up for the desired time range and calculate the average.
//
// The night-value of the last day is always 0, because the night is not complete (day is
// over at 24:00 and the night lasts til 6:00)
//
//  params:
//      db:             Database
//      opt:            different options (see further down)
//
//  return
//      JSON
//      { sid: 29212, span: 30, start: "2019-09-29", count: 30, values: [
//          { date: "2019-09-29", n_dayAVG: 49.45592437272605, n_nightAVG: 53.744277577490614 },
//          { date: "2019-09-30", n_dayAVG: 51.658169450663465, n_nightAVG: 47.82407695888631 },
//              .........
//      ]}
//      CSV
//      datetime,n_dayAVG,n_nightAVG
//      2019-09-29,49.45592437272605,53.744277577490614
//      2019-09-30,51.658169450663465,47.82407695888631
//          ....
//
// *********************************************
async function getdaynightData(opts) {
    opts.long = true;
    let erg = await getNoiseAVGData(opts);
    let val = [];
    let csvStr = 'datetime,n_dayAVG,n_nightAVG\n';
    if (!erg.err) {
        let done = false;
        let dt;
        // The received hourly data array always (!!) starts at 0h00 local (!) time.
        // So to calculate day values, we skip the first 6 hour and start from there
        // now we add 16 hour for day and following 8 hour for night
        for (let i = 6; i < erg.values.length;) {
            let dsum = 0, dcnt = 0;
            let nsum = 0, ncnt = 0;
            let werte = {};
            for (let k = 0; k < 16; k++, i++) {
                const item = erg.values[i]
                if (item.n_sum != -1) {
                    if (werte.datetime === undefined) {
                        let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                        werte.datetime = dt.startOf('day').toISO()
                    }
                    dsum += item.n_sum;
                    dcnt += item.count;
                }
            }
            if (i < (erg.values.length - 8)) {
                const item = erg.values[i]
                for (let k = 0; k < 8; k++, i++) {
                    if (item.n_sum != -1) {
                        if (werte.datetime === undefined) {
                            let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                            werte.datetime = dt.startOf('day').toISO()
                        }
                        nsum += item.n_sum;
                        ncnt += item.count;
                    }
                }
            } else {
                done = true;
            }
            if (dcnt != 0) {
                werte.n_dayAVG = 10 * Math.log10(dsum / dcnt);
            } else {
                werte.n_dayAVG = 0;
            }
            if (ncnt != 0) {
                werte.n_nightAVG = 10 * Math.log10(nsum / ncnt);
            } else {
                werte.n_nightAVG = 0;
            }
            if (opts.csv) {
                csvStr += werte.datetime + ',' + werte.n_dayAVG + ',' + werte.n_nightAVG + '\n'
            } else {
                val.push(werte);
            }
            if (done) {
                break;
            }
        }
    }
    if (opts.csv) {
        return csvStr;
    }
    return {err: erg.err, values: val}
}


// *********************************************
// getLdenData
//
// Use hour averages to calculate the LDEN.
// Formula:
// LDEN = 10 * log10 ( 1/24 ( (12 * 10^(Lday/10)) + (4*10^((Levn+5)/10) + (8*10^((Lnight+10)/10)) )
//
//  params:
//      db:             Database
//      sid:            sensor number
//      opt:            different options (see further down)
//
//  return:
//      JSON:
//      { sid: 29212, span: 30, start: "2019-09-29", count: 30, values: [
//          { lden: 59.53553743437777, date: "2019-09-29" },
//          { lden: 55.264733497513554, date: "2019-09-30" },
//              .........
//      ]}
//      CSV
//      datetime,lden
//      2019-09-29,59.53553743437777
//      2019-09-30,55.264733497513554
//          ....
//
// *********************************************
async function getLdenData(opts) {
    opts.long = true;
    let erg = await getNoiseAVGData(opts);
    let val = [];
    let csvStr = 'datetime,lden\n';
    if (!erg.err) {
        let done = false;
        const calcAVG = (sum, cnt) => {
            if (cnt != 0) {
                return (10 * Math.log10(sum / cnt));
            } else {
                return 0;
            }
        }

        // The received hourly data array always (!!) starts at 0h00 local (!) time.
        // So to calculate day values, we skip the first 6 hour and start from there
        // now we add 12 hour for day and following 4 hour for evening and
        // additional 8 hours for night
        for (let i = 6; i < erg.values.length;) {
            let dsum = 0, dcnt = 0;
            let nsum = 0, ncnt = 0;
            let esum = 0, ecnt = 0;
            let werte = {};
            let dayAVG = 0, evnAVG = 0, nightAVG = 0;
            for (let k = 0; k < 12; k++, i++) {
                const item = erg.values[i]
                if (item.n_sum != -1) {
                    if (werte.datetime == undefined) {
                        let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                        werte.datetime = dt.startOf('day').toISO()
                    }
                    dsum += item.n_sum;
                    dcnt += item.count;
                }
            }
            for (let k = 0; k < 4; k++, i++) {
                const item = erg.values[i]
                if (item.n_sum != -1) {
                    if (werte.datetime == undefined) {
                        let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                        werte.datetime = dt.startOf('day').toISO()
                    }
                    esum += item.n_sum;
                    ecnt += item.count;
                }
            }
            if (i < (erg.values.length - 8)) {
                for (let k = 0; k < 8; k++, i++) {
                    const item = erg.values[i]
                    if (item.n_sum != -1) {
                        if (werte.datetime == undefined) {
                            let dt = DateTime.fromISO(item.datetime, {zone: 'utc'})
                            werte.datetime = dt.startOf('day').toISO()
                        }
                        nsum += item.n_sum;
                        ncnt += item.count;
                    }
                }
            } else {
                done = true;
            }
            dayAVG = calcAVG(dsum, dcnt);
            evnAVG = calcAVG(esum, ecnt);
            nightAVG = calcAVG(nsum, ncnt);
            // Calculate LDEN:
            let day = 12 * Math.pow(10, dayAVG / 10);  // ... and calculate the LDEN values following ...
            let evn = 4 * Math.pow(10, (evnAVG + 5) / 10);  // ... the LDEN formaula (see function description)
            let night = 8 * Math.pow(10, (nightAVG + 10) / 10);
            werte.lden = 10 * Math.log10((day + evn + night) / 24);
            if (opts.csv) {
                csvStr += werte.datetime + ',' + werte.lden + '\n'
            } else {
                val.push(werte);
            }
            if (done) {
                break;
            }
        }
    }
    if (opts.csv) {
        return csvStr;
    }
    return {err: erg.err, values: val}
}


const getAPIprops = (opt) => {

}

const getNoiseAVGData = async (opts) => {
    let ret = {err: null, values: []}
    let emptyValues = {n_AVG: -1}
    let small =  '|> keep(columns: ["_time", "peakcount", "n_AVG"])'
    if (opts.long) {
        small = ''
        emptyValues.n_sum = -1
    }
    let queryAVG = `
import "math"
threshold = ${opts.peak}

data = from(bucket: "sensor_data")
  |> range(${opts.start}, ${opts.stop})  
  |> filter(fn: (r) => r["sid"] == "${opts.sensorid}")
e10 = data 
  |> filter(fn: (r) => r._field == "E10tel_eq") 
  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
  |> map(fn:  (r) => ({r with _value: (10.0 * math.log10(x: r._value))}))
  |> keep(columns: ["_time","_field","_value"]) 
  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
  |> rename(columns: {"E10tel_eq" : "n_AVG"})
ecnt = data
  |> filter(fn: (r) => r._field == "E10tel_eq") 
  |> aggregateWindow(every: 1h, fn: count, createEmpty: false)
  |> keep(columns: ["_time","_field","_value"]) 
  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
  |> rename(columns: {"E10tel_eq" : "count"})
esum = data
  |> filter(fn: (r) => r._field == "E10tel_eq") 
  |> aggregateWindow(every: 1h, fn: sum, createEmpty: false)
  |> keep(columns: ["_time","_field","_value"]) 
  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
  |> rename(columns: {"E10tel_eq" : "n_sum"})
peak = data
  |> filter(fn: (r) => r._field == "noise_LA_max") 
  |> aggregateWindow(
        every: 1h,
        fn: (column, tables=<-) => tables
            |> reduce(
                identity: {peakcount: 0.0},
                fn: (r, accumulator) => ({
                    peakcount: if r._value >= threshold then
                        accumulator.peakcount + 1.0
                    else
                        accumulator.peakcount + 0.0,
                }),
            ),
      )
  |> keep(columns: ["_time","peakcount"]) 
  part1 = join( tables: {e10: e10, ecnt: ecnt}, on: ["_time"])
  part2 = join( tables: {esum: esum, peak: peak}, on: ["_time"])
  join( tables: {P1: part1, P2: part2}, on: ["_time"])
  ${small}
`
    ret = await fetchFromInflux(ret, queryAVG)
    if(ret.err) {
        return returnOnError(ret, ret.err, getNoiseAVGData.name)
    }
    if(ret.values.length === 0) {
        return returnOnError(ret, ERR.NODATA, getNoiseAVGData.name)
    }
    // The times are always the END of the period (so: period from 00:00h to 01:00h -> time is 01:00)

    // To easily extract the values, we copy the data from docs into a new array, so that the
    // hour in an element in docs becomes the index into the new array (for every new day this
    // index will be  incremented by 24). Missing values are marked by: {n_sum=-1, n_AVG=-1}.
    let hoursArr = new Array(opts.span * 24);    // generate new array
    hoursArr.fill(emptyValues)                              // fill array with 'empty' values
    let startDay = DateTime.fromISO(ret.values[0].datetime, {zone: 'utc'}).get('day') // calc first day
    let k = 0
    for (let d of ret.values) {                                   // loop through docs
        let stunde = DateTime.fromISO(d.datetime, {zone: 'utc'}).get('hour')  // get current hour
        let day = DateTime.fromISO(d.datetime, {zone: 'utc'}).get('day')  // get current day
        if (day != startDay) {                              // if date has changed
            k += 24                                         // increment index by 24
            startDay = day
        }
        hoursArr[k+stunde] = d                              // copy date into hourArray
    }
    return { err: ret.err, values: hoursArr}
}

const whatTable = [
    {'what':'live', 'span': 1, 'daystart': false, peak: false, 'func': getLiveData},
    {'what':'havg', 'span': 3, 'daystart': false, peak: true, 'func': gethavgData},
    {'what':'davg', 'span': 30, 'daystart': true, peak: true, 'func': getdavgData},
    {'what':'daynight', 'span': 30, 'daystart': true, peak: false, 'func': getdaynightData},
    {'what':'lden', 'span': 30, 'daystart': true, peak: false, 'func': getLdenData},
    {'what':'props', 'span': 0, 'daystart': true, peak:false, 'func': getAPIprops},
];