admin/laermsensor-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

readin/readarchive: gemeinsame Module, Stack-Konventionen

Browse Source 
mongo.js, influx_post.js und logit.js lagen in beiden Komponenten doppelt
und waren auseinandergelaufen - so ist der Zeitzonen-Fehler entstanden, den
readin seit jeher richtig loest und readFromcsv.js zwei Jahre lang nicht.
Sie liegen jetzt einmal unter common/.

Wo die Fassungen sich widersprachen:
- MONGOAUTH wird als String verglichen. Die readarchive-Fassung pruefte nur
  auf truthy, dadurch schaltete auch MONGOAUTH=false die Auth ein.
- writeDataArray(client, coll, data) nimmt den Collection-Namen direkt;
  readarchive baut ihn mit dem neuen dataCollName(styp).
- getallProperties liefert das Array selbst. Die readarchive-Fassung mit
  {error, properties} war dort ungenutzt.
- properties_collection heisst einheitlich property_coll.

Das Sammelobjekt statistics lag in readin/readdata.js und wurde von
mongo.js und influx_post.js importiert. Es liegt jetzt in
common/statistics.js, damit die gemeinsamen Module readin nicht kennen
muessen.

Der Build-Kontext beider Images ist dadurch das Repository-Wurzelverzeichnis;
das Layout im Image spiegelt das Repository, damit ../common/... unveraendert
aufgeht. Die Volume-Zeile von readin im Compose zieht deshalb auf
/opt/app/readin/data um.

readarchive folgt jetzt den Konventionen des Stacks: deploy.sh statt
build_and_copy.sh, Dockerfile_readarchive statt Dockerfile_rfcsv, und die
eigene docker-compose.yml entfaellt - der Dienst haengt als Profil "tools"
im Compose des Stacks und wird mit Parametern gestartet:

  docker compose run --rm readarchive -t noise -s 2026-07-25 -e 2026-07-29

Getestet: beide Images gebaut, beide gegen Testdatenbanken laufen lassen.
Nicht deployt - die Images in der Registry sind unveraendert.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Reinhard X. Fürst
2026-07-31 07:25:54 +00:00
parent 7c6692c040
commit bd5cf9978d
26 changed files with 304 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files
readin/.dockerignore
-10
View File
@@ -1,10 +0,0 @@
node-modules
.gitignore
.dockerignore
build_and_copy.sh
docker-compose.yml
Dockerfile*
INFLUXSERVER
TRANSFERE
data
readin/Dockerfile_readin
+11 -8
View File
@@ -1,14 +1,20 @@
FROM node:22-alpine
ADD package.json /tmp/package.json
ADD package-lock.json /tmp/package-lock.json
# Build-Kontext ist das Repository-Wurzelverzeichnis (siehe deploy.sh), weil
# readin und readarchive sich die Module unter common/ teilen. Das Layout im
# Image spiegelt das Repository, damit die relativen Importe (../common/...)
# unveraendert funktionieren.
ADD readin/package.json /tmp/package.json
ADD readin/package-lock.json /tmp/package-lock.json
RUN cd /tmp && npm ci
RUN mkdir -p /opt/app && cp -a /tmp/node_modules /tmp/package.json /opt/app/
WORKDIR /opt/app
ADD . /opt/app
ADD crontab.tmp /opt/app
ADD common/*.js /opt/app/common/
ADD readin/*.js /opt/app/readin/
ADD readin/package.json readin/crontab.tmp /opt/app/readin/
WORKDIR /opt/app/readin
RUN mkdir -p data
#RUN apk add busybox-initscripts
@@ -23,6 +29,3 @@ RUN touch cmds.sh \
&& echo 'crond -f' >>cmds.sh
CMD sh ./cmds.sh
readin/crontab.tmp
+1 -2
View File
@@ -1,2 +1 @@
3-59/5 * * * * cd /opt/app && npm start
3-59/5 * * * * cd /opt/app/readin && npm start
readin/deploy.sh
+7 -1
View File
@@ -5,15 +5,20 @@ set -e
# Angepasst aus /Projekte/Logbuch/deploy.sh
# Unterschiede: nur linux/amd64, nativer Build auf esprimo (amd64) -> kein buildx/qemu,
# eigener Dockerfile-Name (Dockerfile_readin).
# Build-Kontext ist das Repository-Wurzelverzeichnis, weil readin und
# readarchive sich die Module unter common/ teilen.
REGISTRY="docker.citysensor.de"
IMAGE_NAME="readin"
TAG="${1:-latest}"
PLATFORM="linux/amd64"
DOCKERFILE="Dockerfile_readin"
DOCKERFILE="readin/Dockerfile_readin"
FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}"
BUILD_DATE=$(date +%d.%m.%Y)
# ins Repository-Wurzelverzeichnis wechseln (eine Ebene ueber diesem Skript)
cd "$(dirname "$0")/.."
echo "=========================================="
echo "readin Deploy Script (amd64)"
echo "=========================================="
@@ -22,6 +27,7 @@ echo "Image: ${IMAGE_NAME}"
echo "Tag: ${TAG}"
echo "Platform: ${PLATFORM}"
echo "Dockerfile: ${DOCKERFILE}"
echo "Kontext: $(pwd)"
echo "Build-Datum: ${BUILD_DATE}"
echo "=========================================="
echo ""
readin/fetchnewdata.js
+5 -4
View File
@@ -16,12 +16,13 @@
const TYP = process.env.TYP || ''
const STORE = (process.env.STORE || 'both').toLowerCase() // 'mongo' | 'influx' | 'both'
import { doReadfromAPI as readin, statistics } from './readdata.js'
import { doReadfromAPI as readin } from './readdata.js'
import { constructDBaseEntries as parse} from './parse.js'
import * as mongo from './mongo.js'
import * as influx from './influx_post.js'
import { logit, logerror } from './logit.js'
import * as mongo from '../common/mongo.js'
import * as influx from '../common/influx_post.js'
import { logit, logerror } from '../common/logit.js'
import { statistics } from '../common/statistics.js'
import { DateTime } from 'luxon'
import fs from 'fs'
import mod_getopt from 'posix-getopt'
readin/influx_post.js
-100
View File
@@ -1,100 +0,0 @@
// Access to influxDB vie HTTP
import axios from 'axios'
import { logit, logerror } from './logit.js'
import { DateTime } from 'luxon'
import { statistics } from'./readdata.js'
let DEVELOP = process.env.DEVELOP || 'false'
let INFLUXHOST = process.env.INFLUXHOST || "localhost"
let INFLUXPORT = process.env.INFLUXPORT || 8086
let INFLUXTOKEN = process.env.INFLUXTOKEN || 'empty'
let INFLUXDATABUCKET = process.env.INFLUXDATABUCKET || "sensor_data"
let INFLUXORG = process.env.INFLUXORG || "citysensor"
const INFLUXURL_READ = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/query?org=${INFLUXORG}`
const INFLUXURL_WRITE = `http://${INFLUXHOST}:${INFLUXPORT}/api/v2/write?org=${INFLUXORG}&bucket=${INFLUXDATABUCKET}&precision=ms`
export const influxRead = async (query) => {
let start = DateTime.now()
let data = []
try {
let ret = await axios({
method: 'post',
url: INFLUXURL_READ,
data: query,
headers: {
Authorization: `Token ${INFLUXTOKEN}`,
Accept: 'application/csv',
'Content-type': 'application/vnd.flux'
},
timeout: 10000,
})
if (ret.status != 200) {
logerror(`doReadfromAPI Status: ${ret.status}`)
}
data = ret.data
} catch (e) {
logerror(`doReadfromAPI ${e}`)
}
logit(`ReadIn-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
return data
}
export const influxWrite = async (data) => {
let start = DateTime.now()
let ret
logit(INFLUXURL_WRITE)
if (DEVELOP === 'true') {
logit(`Token: ${INFLUXTOKEN}`)
}
try {
ret = await axios({
method: 'post',
url: INFLUXURL_WRITE,
data: data,
headers: {
Authorization: `Token ${INFLUXTOKEN}`,
Accept: 'application/json',
'Content-Type': 'text/plain; charset=utf-8'
},
timeout: 10000,
})
if (ret.status != 204) {
logerror(`doWrite2API Status: ${ret.status}`)
}
} catch (e) {
logerror(`doWrite2API ${e}`)
}
let statname = `writeInfluxData[sensor_data]Time`
statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
logit(`Influx-Write-Time: ${start.diffNow('seconds').toObject().seconds * -1} sec`)
return ret
}
/*
async function main() {
let data = `
pm,sid=140 P1=12,P2=13
pm,sid=142 P1=42,P2=13
pm,sid=143 P1=43,P2=13
pm,sid=144 P1=44,P2=13
thp,sid=141 temperature=23.5,humidity=48,pressure=998
`
let ret = await influxWrite(data)
process.exit()
let query = `from(bucket:"sensor_data")
|> range(start: -1mo)
|> filter(fn: (r) => r._measurement == "pm")
|> filter(fn: (r) => r.sid == "140")
`
let erg = await influxRead(query)
console.log(erg)
}
main().catch(console.error)
*/
readin/logit.js
-12
View File
@@ -1,12 +0,0 @@
import { DateTime} from 'luxon'
export function logit(str) {
let s = `${DateTime.now().toISO()} => ${str}`;
console.log(s);
}
export function logerror(str) {
let s = `${DateTime.utc().toISO()} => *** ERROR *** ${str}`;
console.log(s);
}
readin/mongo.js
-202
View File
@@ -1,202 +0,0 @@
/* Interface for MongoDB
*/
import { MongoClient } from 'mongodb'
import { logit, logerror } from './logit.js'
import { statistics } from './readdata.js'
import { DateTime } from 'luxon'
let DEVELOP = process.env.DEVELOP || 'false'
let MONGOHOST = process.env.MONGOHOST || 'localhost'
let MONGOPORT = process.env.MONGOPORT || 27017
let MONGOAUTH = process.env.MONGOAUTH || 'false'
let MONGOUSRP = process.env.MONGOUSRP || ''
let MONGOBASE = process.env.MONGOBASE || 'sensor_data'
let MONGO_URL = 'mongodb://' + MONGOHOST + ':' + MONGOPORT; // URL to mongo database
if (MONGOAUTH == 'true') {
MONGO_URL = 'mongodb://' + MONGOUSRP + '@' + MONGOHOST + ':' + MONGOPORT + '/?authSource=admin'; // URL to mongo database
}
export const property_coll = 'properties'
const addandshowstatistics = (client, text, field, start) => {
statistics[field] = DateTime.now().diff(start, ['seconds']).toObject().seconds
logit(`Write ${text} to mongoDB: Time: ${statistics[field]} sec.`)
}
export const connectMongo = async () => {
try {
if(DEVELOP === 'true') {
logit(`Try to connect to ${MONGO_URL}`)
} else {
logit(`Try to connect to ${'mongodb://' + MONGOHOST + ':' + MONGOPORT}`)
}
let client = await MongoClient.connect(MONGO_URL, { useNewUrlParser: true, useUnifiedTopology: true })
if ( DEVELOP === 'true') {
logit(`Mongodbase connected to ${MONGO_URL}`)
} else {
logit('Mongodbase connected')
}
return client
}
catch (error) {
throw (error)
}
}
export const closeMongo = async (client) => {
client.close()
}
export const getallProperties = async (client) => {
return await client.db(MONGOBASE).collection(property_coll)
.find().sort({ _id: 1 }).toArray()
}
export const checkOneproperty = async (client, sid) => {
return await client.db(MONGOBASE).collection("properties")
.findOne({ _id: sid })
}
export const writeOneproperty = async (client, prop) => {
try {
let result = await client.db(MONGOBASE).collection("properties")
.insertOne(prop)
} catch (e) {
if (e.code == 11000) {
return false
} else {
throw (e)
}
}
return true
}
export const writeProperties = async (client, props) => {
let result
let startAll = DateTime.now();
let start
let coll = client.db(MONGOBASE).collection("properties")
if (props.new.length !== 0) {
start = DateTime.now();
try {
result = await coll.insertMany(props.new)
} catch (e) {
logerror(`Write properties new ${e}`)
}
logit(`Write ${props.new.length} properties NEW to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
}
if (props.loc.length !== 0) {
start = DateTime.now()
try {
for (let item of props.loc) {
result = await coll.updateOne({ _id: item._id }, {
$set: { location_id: item.location_id },
$push: { location: { $each: [item.location[0]], $position: 0 } }
})
}
} catch (e) {
logerror(`Write properties location ${e}`)
}
logit(`Write ${props.loc.length} properties LOC to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
}
if (props.sname.length !== 0) {
start = DateTime.now()
try {
for (let item of props.sname) {
result = await coll.updateOne({ _id: item._id }, {
$push: { name: { $each: [item.name[0]], $position: 0 } }
})
}
} catch (e) {
logerror(`Write properties samename ${e}`)
}
logit(`Write ${props.sname.length} properties NAME to mongoDB: Result: ${result.acknowledged}, Time: ${start.diffNow('seconds').toObject().seconds * -1} sec.`)
}
addandshowstatistics(client, 'properties', 'writePropsTime', startAll)
}
export const writeDataArray = async (client, coll, data) => {
let result
let start = DateTime.now();
try {
result = await client.db(MONGOBASE).collection(coll)
.insertMany(data, { ordered: false })
} catch (e) {
if (e.code !== 11000) {
console.error(e)
}
}
let statname = `writeMongoData[${coll}]Time`
addandshowstatistics(client, `${data.length} entries for ${coll}`, `writeMongoData[${coll}]Time`, start)
// statistics[statname] = DateTime.now().diff(start, ['seconds']).toObject().seconds
// logit(`Write Data for ${coll} to mongoDB: Time: ${statistics[statname]} sec.`)
}
export const writeStatistic = async (client, stat) => {
let result
let start = DateTime.now();
let entry = { timestamp: new Date(), ...stat }
try {
result = await client.db(MONGOBASE).collection("statistics")
.insertOne(entry)
} catch (e) {
console.error(e)
}
addandshowstatistics(client, `statistics`, `writeStatisticTime`, start)
}
export const dropColl = async (client, coll) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection(coll).drop()
} catch (e) {
console.error(e)
}
logit(`Drop collection ${coll}: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
}
export const createIndex = async (client, coll) => {
let result
let start = DateTime.now()
try {
result = await client.db(MONGOBASE).collection(coll).createIndex({ "location.loc": "2dsphere" })
} catch (e) {
console.error(e)
}
logit(`Create-Index: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
}
export const bulkUpdateMapdata = async (client, data) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection("mapdata")
.bulkWrite(data, { ordered: false })
} catch (e) {
console.error(e)
}
logit(`Write MapData: Result: ${result}, Time: ${start.diffNow('second').toObject().seconds * -1} sec.`)
return result
}
export const bulkWrite = async (client, coll, data) => {
let start = DateTime.now()
let result
try {
result = await client.db(MONGOBASE).collection(coll)
.bulkWrite(data, { ordered: false })
} catch (e) {
console.error(e)
}
addandshowstatistics(client, `Data for ${coll}`, `writeMongoProperties[${coll}]Time`, start)
return result
}
readin/parse.js
+3 -3
View File
@@ -1,8 +1,8 @@
// import logit from './logit.js'
import { DateTime } from 'luxon'
import * as mongo from './mongo.js'
import { statistics } from'./readdata.js'
import { logit, logerror } from './logit.js'
import * as mongo from '../common/mongo.js'
import { statistics } from'../common/statistics.js'
import { logit, logerror } from '../common/logit.js'
let actualProps = []
let newProps = []
readin/readdata.js
+2 -2
View File
@@ -1,12 +1,12 @@
import axios from 'axios'
import * as fs from 'fs'
import { logit, logerror} from'./logit.js'
import { logit, logerror} from'../common/logit.js'
import { statistics } from '../common/statistics.js'
import { DateTime } from 'luxon'
const API_URL = 'https://api.sensor.community/static/v1/data.json'; // URL to API on 'luftdaten.info'
const SAVE_NAME = './data/aktdata.json'; // filename for actual data
let LIVE = process.env.LIVE || 'true'
export let statistics = {};
export const doReadfromAPI = async () => {
logit(`LIVE = ${LIVE}`)