diff --git a/.gitignore b/.gitignore index 928ac45..67cd145 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ yarn.lock modules/ui/package-lock.json +logs/ diff --git a/README.md b/README.md index 871f6a8..a6bd52a 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ app_1 | [2018-08-13 12:54:44,264] ERROR in __init__: BUZZER not working app_1 | (1) wsgi starting up on http://0.0.0.0:5000 ``` -The contents of this folder will be mounted to `/usr/src/craftbeerpi3` and the server will be accesible on `localhost:5000`. +The contents of this folder will be mounted to `/usr/src/craftbeerpi3` and the server will be accesible on `localhost:3000`. ## Donation diff --git a/config/logger.yaml b/config/logger.yaml new file mode 100644 index 0000000..4b28e6f --- /dev/null +++ b/config/logger.yaml @@ -0,0 +1,19 @@ +version: 1 +formatters: + simple: + format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s' +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: simple + stream: ext://sys.stdout + file: + class : logging.handlers.RotatingFileHandler + formatter: simple + filename: ./logs/app.log + maxBytes: 10000000 + backupCount: 3 +root: + level: DEBUG + handlers: [console, file] \ No newline at end of file diff --git a/install.sh b/install.sh index 82193cb..195e05a 100755 --- a/install.sh +++ b/install.sh @@ -20,7 +20,8 @@ show_menu () { "8" "Reset File Changes (git reset --hard)" \ "9" "Clear all logs" \ "10" "Reboot Raspberry Pi" \ - "11" "Stop CraftBeerPi, Clear logs, Start CraftBeerPi" 3>&1 1>&2 2>&3) + "11" "Stop CraftBeerPi, Clear logs, Start CraftBeerPi" \ + "12" "Install KairosDB" 3>&1 1>&2 2>&3) BUTTON=$? # Exit if user pressed cancel or escape @@ -49,7 +50,7 @@ show_menu () { apt-get -y install libpcre3-dev pip install -r requirements.txt - confirmAnswer "Would you like to add active 1-wire support at your Raspberry PI now? IMPORTANT: The 1-wire thermometer must be conneted to GPIO 4!" + confirmAnswer "Would you like to add active 1-wire support at your Raspberry PI now? IMPORTANT: The 1-wire thermometer must be connected to GPIO 4!" if [ $? = 0 ]; then #apt-get -y update; apt-get -y upgrade; echo '# CraftBeerPi 1-wire support' >> "/boot/config.txt" @@ -62,14 +63,14 @@ show_menu () { sed "s@#DIR#@${PWD}@g" config/craftbeerpiboot > /etc/init.d/craftbeerpiboot chmod 755 /etc/init.d/craftbeerpiboot; - whiptail --title "Installition Finished" --msgbox "CraftBeerPi installation finished! You must hit OK to continue." 8 78 + whiptail --title "Installation Finished" --msgbox "CraftBeerPi installation finished! You must hit OK to continue." 8 78 show_menu ;; 2) confirmAnswer "Are you sure you want to clear the CraftBeerPi. All hardware setting will be deleted" if [ $? = 0 ]; then sudo rm -f craftbeerpi.db - whiptail --title "Database Delted" --msgbox "The CraftBeerPi database was succesfully deleted. You must hit OK to continue." 8 78 + whiptail --title "Database Deleted" --msgbox "The CraftBeerPi database was successfully deleted. You must hit OK to continue." 8 78 show_menu else show_menu @@ -81,7 +82,7 @@ show_menu () { sed "s@#DIR#@${PWD}@g" config/craftbeerpiboot > /etc/init.d/craftbeerpiboot chmod 755 /etc/init.d/craftbeerpiboot; update-rc.d craftbeerpiboot defaults; - whiptail --title "Added succesfull to autostart" --msgbox "The CraftBeerPi was added to autostart succesfully. You must hit OK to continue." 8 78 + whiptail --title "Added successful to autostart" --msgbox "The CraftBeerPi was added to autostart successfully. You must hit OK to continue." 8 78 show_menu else show_menu @@ -104,7 +105,7 @@ show_menu () { ;; 6) sudo /etc/init.d/craftbeerpiboot stop - whiptail --title "CraftBeerPi stoped" --msgbox "The software is stoped" 8 78 + whiptail --title "CraftBeerPi stopped" --msgbox "The software is stopped" 8 78 show_menu ;; 7) @@ -118,7 +119,7 @@ show_menu () { fi ;; 8) - confirmAnswer "Are you sure you want to reset all file changes for this git respository (git reset --hard)?" + confirmAnswer "Are you sure you want to reset all file changes for this git repository (git reset --hard)?" if [ $? = 0 ]; then whiptail --textbox /dev/stdin 20 50 <<<"$(git reset --hard)" show_menu @@ -155,6 +156,17 @@ show_menu () { show_menu fi ;; + 12) + confirmAnswer "Are you sure you want to install KairosDB?" + if [ $? = 0 ]; then + wget https://github.com/kairosdb/kairosdb/releases/download/v1.2.1/kairosdb_1.2.1-1_all.deb + sudo dpkg -i kairosdb_1.2.1-1_all.deb + sudo service kairosdb start + show_menu + else + show_menu + fi + ;; esac fi } diff --git a/modules/app_config.py b/modules/app_config.py index 96a5e0c..6552202 100644 --- a/modules/app_config.py +++ b/modules/app_config.py @@ -1,23 +1,16 @@ +from flask import Flask, json, g +from flask_socketio import SocketIO -import json -import sys, os -from flask import Flask, render_template, redirect, json, g - - -from flask_socketio import SocketIO, emit - -import logging - - +import yaml +import logging.config from modules.core.core import CraftBeerPi, ActorBase, SensorBase from modules.core.db import DBModel app = Flask(__name__) -FORMAT = '%(asctime)-15s - %(levelname)s - %(message)s' +logging.config.dictConfig(yaml.load(open('./config/logger.yaml', 'r'))) -logging.basicConfig(filename='./logs/app.log',level=logging.INFO, format=FORMAT) app.config['SECRET_KEY'] = 'craftbeerpi' app.config['UPLOAD_FOLDER'] = './upload' diff --git a/modules/core/core.py b/modules/core/core.py index ff6dfe6..2505718 100644 --- a/modules/core/core.py +++ b/modules/core/core.py @@ -1,16 +1,14 @@ import inspect -import pprint -import sqlite3 from flask import make_response, g import datetime from datetime import datetime -from flask.views import MethodView -from flask_classy import FlaskView, route from time import localtime, strftime from functools import wraps, update_wrapper +import requests +import json from props import * @@ -18,6 +16,7 @@ from hardware import * import time import uuid +import logging class NotificationAPI(object): @@ -86,6 +85,8 @@ class SensorAPI(object): :return: ''' + self.logger = logging.getLogger(__name__) + self.app.logger.info("Init Sensors") t = self.cache.get("sensor_types") @@ -145,22 +146,48 @@ class SensorAPI(object): def receive_sensor_value(self, id, value): self.emit("SENSOR_UPDATE", self.cache.get("sensors")[id]) - self.save_to_file(id, value) + self.save_to_file(id, value, "sensor") + + def save_to_file(self, id, value, prefix): + sensor_name = "%s_%s" % (prefix, str(id)) + use_kairosdb = (self.cache["config"]["kairos_db"].__dict__["value"] == "YES") + + if use_kairosdb: + self.write_to_tsdb(prefix, sensor_name, value) + else: + self.write_to_logfile(sensor_name, value) - def save_to_file(self, id, value, prefix="sensor"): - filename = "./logs/%s_%s.log" % (prefix, str(id)) + def write_to_logfile(self, sensor_name, value): + filename = "./logs/%s.log" % sensor_name formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) - msg = str(formatted_time) + "," +str(value) + "\n" + msg = str(formatted_time) + "," + str(value) + "\n" with open(filename, "a") as file: file.write(msg) + def write_to_tsdb(self, prefix, sensor_name, value): + kairosdb_server = "http://127.0.0.1:" + self.cache["config"]["kairos_db_port"].__dict__["value"] + + data = [ + dict(name="cbpi." + sensor_name, datapoints=[ + [int(round(time.time() * 1000)), value] + ], tags={ + "cbpi": prefix, + "brew": self.cache["active_brew"] + }) + ] + + response = requests.post(kairosdb_server + "/api/v1/datapoints", json.dumps(data)) + if not response.ok: + self.logger.warning("Failed to write time series entry for [%s]. Response [%s]", sensor_name, response) + def log_action(self, text): - filename = "./logs/action.log" - formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime()) - with open(filename, "a") as file: - text = text.encode("utf-8") - file.write("%s,%s\n" % (formatted_time, text)) + use_kairosdb = (self.cache["config"]["kairos_db"].__dict__["value"] == "YES") + + if use_kairosdb: + self.write_to_tsdb("action", "action", text) + else: + self.write_to_logfile("action", text) def shutdown_sensor(self, id): self.cache.get("sensors")[id].stop() @@ -204,7 +231,8 @@ class CraftBeerPi(ActorAPI, SensorAPI): "messages": [], "plugins": {}, "fermentation_controller_types": {}, - "fermenter_task": {} + "fermenter_task": {}, + "active_brew": "none" } buzzer = None eventbus = {} diff --git a/modules/logs/endpoints.py b/modules/logs/endpoints.py index 5f8b29e..4eb7e2b 100644 --- a/modules/logs/endpoints.py +++ b/modules/logs/endpoints.py @@ -1,11 +1,15 @@ import datetime import os -from flask import Blueprint, request, send_from_directory, json +import requests +import logging +from flask import request, send_from_directory, json from flask_classy import FlaskView, route from modules import cbpi class LogView(FlaskView): + def __init__(self): + self.logger = logging.getLogger(__name__) @route('/', methods=['GET']) def get_all_logfiles(self): @@ -17,19 +21,10 @@ class LogView(FlaskView): @route('/actions') def actions(self): - filename = "./logs/action.log" - if os.path.isfile(filename) == False: - return - import csv - array = [] - with open(filename, 'rb') as f: - reader = csv.reader(f) - for row in reader: - try: - array.append([int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000, row[1]]) - except: - pass - return json.dumps(array) + array = self.read_log_as_json("action") + json_dumps = json.dumps(array) + + return json_dumps @route('/', methods=["DELETE"]) def clearlog(self, file): @@ -49,24 +44,85 @@ class LogView(FlaskView): cbpi.notify("Failed to delete log", "", type="danger") return ('', 204) - def read_log_as_json(self, type, id): - filename = "./logs/%s_%s.log" % (type, id) - if os.path.isfile(filename) == False: - return + def query_tsdb(self, sensor_name): + kairosdb_server = "http://127.0.0.1:" + cbpi.cache["config"]["kairos_db_port"].__dict__["value"] + + data = dict(metrics=[ + { + "tags": {}, + "name": "cbpi.%s" % sensor_name, + "aggregators": [ + { + "name": "avg", + "align_sampling": True, + "sampling": { + "value": cbpi.cache["config"]["kairos_db_sampling_value"].__dict__["value"], + "unit": "seconds" + }, + "align_start_time": True + } + ] + } + ], + cache_time=0, + start_relative={ + "value": cbpi.cache["config"]["kairos_db_start_relative"].__dict__["value"], + "unit": "days" + }) + + if cbpi.cache["active_brew"] != "none": + data["metrics"][0]["tags"] = {"brew": [cbpi.cache["active_brew"]]} + + self.logger.debug("query: %s", json.dumps(data)) + + response = requests.post(kairosdb_server + "/api/v1/datapoints/query", json.dumps(data)) + if response.ok: + self.logger.debug("Fetching time series for [%s] took [%s]", sensor_name, response.elapsed) + self.logger.debug("Time series for [%s] is [%s]", sensor_name, response.json()) + return response.json()["queries"][0]["results"][0]["values"] + else: + self.logger.warning("Failed to fetch time series for [%s]. Response [%s]", sensor_name, response) - import csv + def query_log(self, filename, value_type): array = [] + + if not os.path.isfile(filename): + self.logger.warn("File does not exist [%s]", filename) + return array + + import csv + + if value_type == "float": + converter = float + else: + converter = str + with open(filename, 'rb') as f: reader = csv.reader(f) for row in reader: try: - array.append([int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000, float(row[1])]) + point_of_time = int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") + - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000 + value = converter(row[1]) + array.append([point_of_time, value]) except: + self.logger.exception("error in reading logfile [%s]", filename) pass return array + def read_log_as_json(self, sensor_name): + use_kairosdb = (cbpi.cache["config"]["kairos_db"].__dict__["value"] == "YES") + + if use_kairosdb: + return self.query_tsdb(sensor_name) + else: + filename = "./logs/%s.log" % sensor_name + return self.query_log(filename, "float") + + def convert_chart_data_to_json(self, chart_data): - return {"name": chart_data["name"], "data": self.read_log_as_json(chart_data["data_type"], chart_data["data_id"])} + return {"name": chart_data["name"], + "data": self.read_log_as_json(chart_data["data_type"] + "_" + str(chart_data["data_id"]))} @route('//', methods=["POST"]) def get_logs_as_json(self, t, id): @@ -74,7 +130,8 @@ class LogView(FlaskView): result = [] if t == "s": name = cbpi.cache.get("sensors").get(id).name - result.append({"name": name, "data": self.read_log_as_json("sensor", id)}) + sensor_name = "%s_%s" % ("sensor", str(id)) + result.append({"name": name, "data": self.read_log_as_json(sensor_name)}) if t == "k": kettle = cbpi.cache.get("kettle").get(id) @@ -82,7 +139,8 @@ class LogView(FlaskView): if t == "f": fermenter = cbpi.cache.get("fermenter").get(id) - result = map(self.convert_chart_data_to_json, cbpi.get_fermentation_controller(fermenter.logic).get("class").chart(fermenter)) + result = map(self.convert_chart_data_to_json, + cbpi.get_fermentation_controller(fermenter.logic).get("class").chart(fermenter)) return json.dumps(result) @@ -99,6 +157,7 @@ class LogView(FlaskView): return True if pattern.match(name) else False + @cbpi.initalizer() def init(app): """ diff --git a/modules/recipe_import/beerxml.py b/modules/recipe_import/beerxml.py index 5e39d11..262b0c8 100644 --- a/modules/recipe_import/beerxml.py +++ b/modules/recipe_import/beerxml.py @@ -46,13 +46,13 @@ class BeerXMLImport(FlaskView): @route('/', methods=['POST']) def load(self, id): - steps = self.getSteps(id) boil_time_alerts = self.getBoilAlerts(id) name = self.getRecipeName(id) self.api.set_config_parameter("brew_name", name) boil_time = self.getBoilTime(id) mashstep_type = cbpi.get_config_parameter("step_mash", "MashStep") + mashinstep_type = cbpi.get_config_parameter("step_mashin", "MashInStep") mash_kettle = cbpi.get_config_parameter("step_mash_kettle", None) boilstep_type = cbpi.get_config_parameter("step_boil", "BoilStep") @@ -65,10 +65,20 @@ class BeerXMLImport(FlaskView): try: + # Add mash in or mash step, depends on timer > 0 for row in steps: - Step.insert(**{"name": row.get("name"), "type": mashstep_type, "config": {"kettle": mash_kettle, "temp": float(row.get("temp")), "timer": row.get("timer")}}) + if row.get("timer") > 0: + Step.insert(**{"name": row.get("name"), "type": mashstep_type, + "config": {"kettle": mash_kettle, "temp": float(row.get("temp")), + "timer": row.get("timer")}}) + else: + Step.insert(**{"name": row.get("name"), "type": mashinstep_type, + "config": {"kettle": mash_kettle, "temp": float(row.get("temp"))}}) + + # Add chilling step Step.insert(**{"name": "ChilStep", "type": "ChilStep", "config": {"timer": 15}}) - ## Add boiling step + + # Add boiling step Step.insert(**{ "name": "Boil", "type": boilstep_type, @@ -76,11 +86,11 @@ class BeerXMLImport(FlaskView): "kettle": boil_kettle, "temp": boil_temp, "timer": boil_time, - ## Beer XML defines additions as the total time spent in boiling, - ## CBP defines it as time-until-alert + # Beer XML defines additions as the total time spent in boiling, + # CBP defines it as time-until-alert - ## Also, The model supports five boil-time additions. - ## Set the rest to None to signal them being absent + # Also, The model supports five boil-time additions. + # Set the rest to None to signal them being absent "hop_1": boil_time - boil_time_alerts[0] if len(boil_time_alerts) >= 1 else None, "hop_2": boil_time - boil_time_alerts[1] if len(boil_time_alerts) >= 2 else None, "hop_3": boil_time - boil_time_alerts[2] if len(boil_time_alerts) >= 3 else None, @@ -88,7 +98,8 @@ class BeerXMLImport(FlaskView): "hop_5": boil_time - boil_time_alerts[4] if len(boil_time_alerts) >= 5 else None } }) - ## Add Whirlpool step + + # Add Whirlpool step Step.insert(**{"name": "Whirlpool", "type": "ChilStep", "config": {"timer": 15}}) StepView().reset() self.api.emit("UPDATE_ALL_STEPS", Step.get_all()) @@ -141,7 +152,12 @@ class BeerXMLImport(FlaskView): else: temp = round(9.0 / 5.0 * float(e.find("STEP_TEMP").text) + 32, 2) - steps.append({"name": e.find("NAME").text, "temp": temp, "timer": float(e.find("STEP_TIME").text)}) + if e.find("STEP_TIME").text is None: + stepTime = 0.0 + else: + stepTime = float(e.find("STEP_TIME").text) + + steps.append({"name": e.find("NAME").text, "temp": temp, "timer": stepTime}) return steps diff --git a/modules/steps/__init__.py b/modules/steps/__init__.py index 05d65fc..7827c14 100755 --- a/modules/steps/__init__.py +++ b/modules/steps/__init__.py @@ -1,4 +1,5 @@ import time +import datetime from flask import json, request from flask_classy import route @@ -93,6 +94,7 @@ class StepView(BaseView): self.model.reset_all_steps() self.stop_step() cbpi.emit("UPDATE_ALL_STEPS", self.model.get_all()) + cbpi.cache["active_brew"] = "none" return ('', 204) def stop_step(self): @@ -142,9 +144,15 @@ class StepView(BaseView): # set step instance to ache cbpi.cache["active_step"] = instance - @route('/next', methods=['POST']) @route('/start', methods=['POST']) def start(self): + if "none" == cbpi.cache["active_brew"]: + cbpi.cache["active_brew"] = cbpi.cache["config"]["brew_name"].__dict__["value"] + \ + "_" + datetime.datetime.now().strftime('%y-%m-%dT%H:%M') + return self.next() + + @route('/next', methods=['POST']) + def next(self): active = Step.get_by_state("A") inactive = Step.get_by_state('I') @@ -163,6 +171,7 @@ class StepView(BaseView): else: cbpi.log_action("Brewing Finished") cbpi.notify("Brewing Finished", "You are done!", timeout=None) + cbpi.cache["active_brew"] = "none" cbpi.emit("UPDATE_ALL_STEPS", Step.get_all()) return ('', 204) diff --git a/modules/system/endpoints.py b/modules/system/endpoints.py index bfb23b3..0a01a28 100755 --- a/modules/system/endpoints.py +++ b/modules/system/endpoints.py @@ -61,6 +61,10 @@ class SystemView(FlaskView): for t in repo.tags: tags.append({"name": t.name, "commit": str(t.commit), "date": t.commit.committed_date, "committer": t.commit.committer.name, "message": t.commit.message}) + for b in repo.branches: + tags.append({"name": b.name, "commit": str(b.commit), "date": b.commit.committed_date, + "committer": b.commit.committer.name, "message": b.commit.message}) + try: branch_name = repo.active_branch.name # test1 diff --git a/update/4_kairosdb_config.sql b/update/4_kairosdb_config.sql new file mode 100644 index 0000000..ce824a1 --- /dev/null +++ b/update/4_kairosdb_config.sql @@ -0,0 +1,4 @@ +INSERT OR IGNORE INTO config VALUES ('kairos_db', 'NO', 'select', 'Use timeseries database KairosDB for storing sensor values. You can install KairosDB with the CraftBeerPi installer.', '["YES","NO"]' ); +INSERT OR IGNORE INTO config VALUES ('kairos_db_port', 8080, 'number', 'Port for KairosDB. We assume the DB is running on your PI, so IP-Address is 127.0.0.1.', NULL ); +INSERT OR IGNORE INTO config VALUES ('kairos_db_sampling_value', 5, 'number', 'A timeseries database has the advantage to aggregate data points and therefore to reduce the transmitted data. This value sets a time span in seconds to calculate the average', NULL ); +INSERT OR IGNORE INTO config VALUES ('kairos_db_start_relative', 1, 'number', 'If you have an ongoing brew or fermentation process only values related to this process will be shown in the graph. Additionally you can define in days how far the time series should reach in the past. The earliest time of both information defines when the data series starts.', NULL ); \ No newline at end of file