tomru/cram-luftdaten
1
0
Fork 0
mirror of https://github.com/tomru/cram-luftdaten.git synced 2026-03-02 22:17:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

get rid of linting warnings

Browse Source
This commit is contained in:
Thomas Ruoff
2019-09-12 20:45:57 +02:00
parent c184f7aaa6
commit 9d31b84e6b
4 changed files with 93 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -7,11 +7,10 @@ You can use this to fill up your local InfluxDB with previous data or fill data
# How does it work
* get the relevant CSVs from https://www.madavi.de/sensor/csvfiles.php (dates
are in UTC!)
* delete all lines that you already have in your DB (you can keep the csv header line)
* get the relevant CSVs from https://www.madavi.de/sensor/csvfiles.php
* delete all lines that you already have in your DB (you must keep the csv header line)
* export settings as environment vars in case they differ from the defaults
(INFLUXDB_HOST, INFLUXDB_PORT, INFLUXDB_DATABASE, SENSOR_ID)
* run `cat <file1.csv> [file2.csv...] | ./toLineProtocol.py | ./toInfluxDb.sh`
* run `cat <file1.csv> [file2.csv...] | ./to_line_protocol.py | ./to_influx_db.sh`
I should advise you to backup the DB first :stuck_out_tongue:

84
toLineProtocol.py
View File

@@ -1,84 +0,0 @@
#!/usr/bin/env python3
#
# Converts CSV exports from https://www.madavi.de/sensor/csvfiles.php to
# InfluxDB LineProtocol https://docs.influxdata.com/influxdb/v1.2/write_protocols/line_protocol_reference/
#
# Note: timestamps are in seconds, therefore precision "s" needs to be set
# when writing, see https://docs.influxdata.com/influxdb/v1.2/tools/api/#write
#
# Settings:
#
# Please set
# * SENSOR_ID
# * INFLUXDB_DATABASE
# in your environment
# CSV file specs
#
# | DB Field | DB field | CSV Column | CSV Format
# |--------------------|----------|--------------------|--------------------------
# | time | time | Time | 2017/05/19 00:00:11 (UTC)
# | ? | ? | durP1 |
# | ? | ? | ratioP1 |
# | ? | ? | P1 |
# | ? | ? | durP2 |
# | ? | ? | ratioP2 |
# | ? | ? | P2 |
# | SDS_P1 | field | SDS_P1 | 10.36
# | SDS_P2 | field | SDS_P2 | 9.50
# | PMS_P1 | field | PMS_P1 |
# | PMS_P2 | field | PMS_P2 |
# | temperature | field | Temp | 16.00
# | humidity | field | Humidity | 79.00
# | BMP_temperature | field | BMP_temperature |
# | BMP_pressure | field | BMP_pressure |
# | BME280_temperature | field | BME280_temperature |
# | BME280_humidity | field | BME280_humidity |
# | BME280_pressure | field | BME280_pressure |
# | samples | field | Samples | 828799
# | min_micro | field | Min_cycle | 172
# | max_micro | field | Max_cycle | 25198
# | signal | field | Signal | -91
# | node | tag | N/A filename? | e.g. esp8266-16229960
import os
import sys
import csv
from datetime import datetime, timedelta
def get_timestamp(timestr):
naive_dt = datetime.strptime(timestr, '%Y/%m/%d %H:%M:%S');
utc = (naive_dt - datetime(1970, 1, 1)) / timedelta(seconds=1)
return int(utc)
SENSOR_ID = os.environ.get('SENSOR_ID', '16229960')
DATABASE = os.environ.get('INFLUXDB_DATABASE', 'sensors')
NODE = 'esp8266-' + SENSOR_ID
NAME_MAP = {
'Humidity': 'humidity',
'Max_cycle': 'max_micro',
'Samples': 'samples',
'Min_cycle': 'min_micro',
'Signal': 'signal',
'Temp': 'temperature'
}
READER = csv.DictReader(sys.stdin, delimiter=';')
for row in READER:
measurements = []
for header, value in row.items():
if header == 'Time' or not value:
continue
measurements.append('{0}={1}'.format(NAME_MAP.get(header, header), value))
values = {
'database': DATABASE,
'node': NODE,
'measurements': ','.join(measurements),
'time': get_timestamp(row['Time'])
}
print('{database},node={node} {measurements} {time}'.format(**values))

0
toInfluxDb.sh → to_influx_db.sh
View File

90
to_line_protocol.py Executable file
View File

@@ -0,0 +1,90 @@
#!/usr/bin/env python3
""" Converts CSV exports from https://www.madavi.de/sensor/csvfiles.php to InfluxDB LineProtocol
Note: timestamps are in seconds, therefore precision "s" needs to be set
when writing, see https://docs.influxdata.com/influxdb/v1.2/tools/api/#write
Settings:
Please set
* SENSOR_ID
* INFLUXDB_DATABASE
in your environment
CSV file spec:
Semiconlos are used as delimiters. First line are the header columns.
Generally the column name is used as a DB field name, but there are some legacy
exceptions like "time" instead of "Time" as listed in the following table.
| DB Field | DB field | CSV Column | CSV Format
|--------------------|----------|--------------------|--------------------------
| time | time | Time | 2017/05/19 00:00:11 (UTC)
| ? | ? | durP1 |
| ? | ? | ratioP1 |
| ? | ? | P1 |
| ? | ? | durP2 |
| ? | ? | ratioP2 |
| ? | ? | P2 |
| SDS_P1 | field | SDS_P1 | 10.36
| SDS_P2 | field | SDS_P2 | 9.50
| PMS_P1 | field | PMS_P1 |
| PMS_P2 | field | PMS_P2 |
| temperature | field | Temp | 16.00
| humidity | field | Humidity | 79.00
| BMP_temperature | field | BMP_temperature |
| BMP_pressure | field | BMP_pressure |
| BME280_temperature | field | BME280_temperature |
| BME280_humidity | field | BME280_humidity |
| BME280_pressure | field | BME280_pressure |
| samples | field | Samples | 828799
| min_micro | field | Min_cycle | 172
| max_micro | field | Max_cycle | 25198
| signal | field | Signal | -91
| node | tag | N/A filename? | e.g. esp8266-16229960
"""
import os
import sys
import csv
from datetime import datetime, timedelta
def get_timestamp(timestr):
"""Converts CSV time value to a UTC timestamp in seconds"""
naive_dt = datetime.strptime(timestr, '%Y/%m/%d %H:%M:%S')
utc = (naive_dt - datetime(1970, 1, 1)) / timedelta(seconds=1)
return int(utc)
SENSOR_ID = os.environ.get('SENSOR_ID', '16229960')
DATABASE = os.environ.get('INFLUXDB_DATABASE', 'sensors')
NODE = 'esp8266-' + SENSOR_ID
NAME_MAP = {
'Humidity':
'humidity',
'Max_cycle': 'max_micro',
'Samples': 'samples',
'Min_cycle': 'min_micro',
'Signal': 'signal',
'Temp': 'temperature'
}
READER = csv.DictReader(sys.stdin, delimiter=';')
for row in READER:
measurements = []
for header, value in row.items():
if header == 'Time' or not value:
continue
measurements.append('{0}={1}'.format(NAME_MAP.get(header, header), value))
values = {
'database': DATABASE,
'node': NODE,
'measurements': ','.join(measurements),
'time': get_timestamp(row['Time'])
}
print('{database},node={node} {measurements} {time}'.format(**values))