From 9d31b84e6bed6daf38f7e9d00e24cd0e599f8d56 Mon Sep 17 00:00:00 2001 From: Thomas Ruoff Date: Thu, 12 Sep 2019 20:45:57 +0200 Subject: [PATCH] get rid of linting warnings --- README.md | 7 ++- toLineProtocol.py | 84 ----------------------------- toInfluxDb.sh => to_influx_db.sh | 0 to_line_protocol.py | 90 ++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 88 deletions(-) delete mode 100755 toLineProtocol.py rename toInfluxDb.sh => to_influx_db.sh (100%) create mode 100755 to_line_protocol.py diff --git a/README.md b/README.md index 03aa566..15df1b2 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,10 @@ You can use this to fill up your local InfluxDB with previous data or fill data # How does it work -* get the relevant CSVs from https://www.madavi.de/sensor/csvfiles.php (dates - are in UTC!) -* delete all lines that you already have in your DB (you can keep the csv header line) +* get the relevant CSVs from https://www.madavi.de/sensor/csvfiles.php +* delete all lines that you already have in your DB (you must keep the csv header line) * export settings as environment vars in case they differ from the defaults (INFLUXDB_HOST, INFLUXDB_PORT, INFLUXDB_DATABASE, SENSOR_ID) -* run `cat [file2.csv...] | ./toLineProtocol.py | ./toInfluxDb.sh` +* run `cat [file2.csv...] | ./to_line_protocol.py | ./to_influx_db.sh` I should advise you to backup the DB first :stuck_out_tongue: diff --git a/toLineProtocol.py b/toLineProtocol.py deleted file mode 100755 index 6e55ee4..0000000 --- a/toLineProtocol.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -# -# Converts CSV exports from https://www.madavi.de/sensor/csvfiles.php to -# InfluxDB LineProtocol https://docs.influxdata.com/influxdb/v1.2/write_protocols/line_protocol_reference/ -# -# Note: timestamps are in seconds, therefore precision "s" needs to be set -# when writing, see https://docs.influxdata.com/influxdb/v1.2/tools/api/#write -# -# Settings: -# -# Please set -# * SENSOR_ID -# * INFLUXDB_DATABASE -# in your environment - -# CSV file specs -# -# | DB Field | DB field | CSV Column | CSV Format -# |--------------------|----------|--------------------|-------------------------- -# | time | time | Time | 2017/05/19 00:00:11 (UTC) -# | ? | ? | durP1 | -# | ? | ? | ratioP1 | -# | ? | ? | P1 | -# | ? | ? | durP2 | -# | ? | ? | ratioP2 | -# | ? | ? | P2 | -# | SDS_P1 | field | SDS_P1 | 10.36 -# | SDS_P2 | field | SDS_P2 | 9.50 -# | PMS_P1 | field | PMS_P1 | -# | PMS_P2 | field | PMS_P2 | -# | temperature | field | Temp | 16.00 -# | humidity | field | Humidity | 79.00 -# | BMP_temperature | field | BMP_temperature | -# | BMP_pressure | field | BMP_pressure | -# | BME280_temperature | field | BME280_temperature | -# | BME280_humidity | field | BME280_humidity | -# | BME280_pressure | field | BME280_pressure | -# | samples | field | Samples | 828799 -# | min_micro | field | Min_cycle | 172 -# | max_micro | field | Max_cycle | 25198 -# | signal | field | Signal | -91 -# | node | tag | N/A filename? | e.g. esp8266-16229960 - - -import os -import sys -import csv -from datetime import datetime, timedelta - -def get_timestamp(timestr): - naive_dt = datetime.strptime(timestr, '%Y/%m/%d %H:%M:%S'); - utc = (naive_dt - datetime(1970, 1, 1)) / timedelta(seconds=1) - return int(utc) - -SENSOR_ID = os.environ.get('SENSOR_ID', '16229960') -DATABASE = os.environ.get('INFLUXDB_DATABASE', 'sensors') - -NODE = 'esp8266-' + SENSOR_ID - -NAME_MAP = { - 'Humidity': 'humidity', - 'Max_cycle': 'max_micro', - 'Samples': 'samples', - 'Min_cycle': 'min_micro', - 'Signal': 'signal', - 'Temp': 'temperature' -} - -READER = csv.DictReader(sys.stdin, delimiter=';') -for row in READER: - measurements = [] - for header, value in row.items(): - if header == 'Time' or not value: - continue - measurements.append('{0}={1}'.format(NAME_MAP.get(header, header), value)) - - values = { - 'database': DATABASE, - 'node': NODE, - 'measurements': ','.join(measurements), - 'time': get_timestamp(row['Time']) - } - - print('{database},node={node} {measurements} {time}'.format(**values)) diff --git a/toInfluxDb.sh b/to_influx_db.sh similarity index 100% rename from toInfluxDb.sh rename to to_influx_db.sh diff --git a/to_line_protocol.py b/to_line_protocol.py new file mode 100755 index 0000000..33f356b --- /dev/null +++ b/to_line_protocol.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +""" Converts CSV exports from https://www.madavi.de/sensor/csvfiles.php to InfluxDB LineProtocol + + Note: timestamps are in seconds, therefore precision "s" needs to be set + when writing, see https://docs.influxdata.com/influxdb/v1.2/tools/api/#write + +Settings: + + Please set + * SENSOR_ID + * INFLUXDB_DATABASE + in your environment + +CSV file spec: + + Semiconlos are used as delimiters. First line are the header columns. + Generally the column name is used as a DB field name, but there are some legacy + exceptions like "time" instead of "Time" as listed in the following table. + + | DB Field | DB field | CSV Column | CSV Format + |--------------------|----------|--------------------|-------------------------- + | time | time | Time | 2017/05/19 00:00:11 (UTC) + | ? | ? | durP1 | + | ? | ? | ratioP1 | + | ? | ? | P1 | + | ? | ? | durP2 | + | ? | ? | ratioP2 | + | ? | ? | P2 | + | SDS_P1 | field | SDS_P1 | 10.36 + | SDS_P2 | field | SDS_P2 | 9.50 + | PMS_P1 | field | PMS_P1 | + | PMS_P2 | field | PMS_P2 | + | temperature | field | Temp | 16.00 + | humidity | field | Humidity | 79.00 + | BMP_temperature | field | BMP_temperature | + | BMP_pressure | field | BMP_pressure | + | BME280_temperature | field | BME280_temperature | + | BME280_humidity | field | BME280_humidity | + | BME280_pressure | field | BME280_pressure | + | samples | field | Samples | 828799 + | min_micro | field | Min_cycle | 172 + | max_micro | field | Max_cycle | 25198 + | signal | field | Signal | -91 + | node | tag | N/A filename? | e.g. esp8266-16229960 + +""" + +import os +import sys +import csv +from datetime import datetime, timedelta + +def get_timestamp(timestr): + """Converts CSV time value to a UTC timestamp in seconds""" + naive_dt = datetime.strptime(timestr, '%Y/%m/%d %H:%M:%S') + utc = (naive_dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + return int(utc) + +SENSOR_ID = os.environ.get('SENSOR_ID', '16229960') +DATABASE = os.environ.get('INFLUXDB_DATABASE', 'sensors') + +NODE = 'esp8266-' + SENSOR_ID + +NAME_MAP = { + 'Humidity': + 'humidity', + 'Max_cycle': 'max_micro', + 'Samples': 'samples', + 'Min_cycle': 'min_micro', + 'Signal': 'signal', + 'Temp': 'temperature' +} + +READER = csv.DictReader(sys.stdin, delimiter=';') +for row in READER: + measurements = [] + for header, value in row.items(): + if header == 'Time' or not value: + continue + measurements.append('{0}={1}'.format(NAME_MAP.get(header, header), value)) + + values = { + 'database': DATABASE, + 'node': NODE, + 'measurements': ','.join(measurements), + 'time': get_timestamp(row['Time']) + } + + print('{database},node={node} {measurements} {time}'.format(**values))