Merge 4d574b1278 into eda209422f

.gitignore

@@ -17,3 +17,4 @@ yarn.lock
 
 
 modules/ui/package-lock.json
+logs/

README.md

@@ -41,7 +41,7 @@ app_1  | [2018-08-13 12:54:44,264] ERROR in __init__: BUZZER not working
 app_1  | (1) wsgi starting up on http://0.0.0.0:5000
 ```
 
-The contents of this folder will be mounted to `/usr/src/craftbeerpi3` and the server will be accesible on `localhost:5000`.
+The contents of this folder will be mounted to `/usr/src/craftbeerpi3` and the server will be accesible on `localhost:3000`.
 
 ## Donation
 

config/logger.yaml

@@ -0,0 +1,19 @@
+version: 1
+formatters:
+  simple:
+    format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class : logging.handlers.RotatingFileHandler
+    formatter: simple
+    filename: ./logs/app.log
+    maxBytes: 10000000
+    backupCount: 3
+root:
+  level: DEBUG
+  handlers: [console, file]

install.sh

@@ -20,7 +20,8 @@ show_menu () {
    "8" "Reset File Changes (git reset --hard)" \
    "9" "Clear all logs" \
    "10" "Reboot Raspberry Pi" \
-   "11" "Stop CraftBeerPi, Clear logs, Start CraftBeerPi" 3>&1 1>&2 2>&3)
+   "11" "Stop CraftBeerPi, Clear logs, Start CraftBeerPi" \
+   "12" "Install KairosDB" 3>&1 1>&2 2>&3)
 
    BUTTON=$?
    # Exit if user pressed cancel or escape
@@ -49,7 +50,7 @@ show_menu () {
            apt-get -y install libpcre3-dev
            pip install -r requirements.txt
 
-           confirmAnswer "Would you like to add active 1-wire support at your Raspberry PI now? IMPORTANT: The 1-wire thermometer must be conneted to GPIO 4!"
+           confirmAnswer "Would you like to add active 1-wire support at your Raspberry PI now? IMPORTANT: The 1-wire thermometer must be connected to GPIO 4!"
            if [ $? = 0 ]; then
              #apt-get -y update; apt-get -y upgrade;
              echo '# CraftBeerPi 1-wire support' >> "/boot/config.txt"
@@ -62,14 +63,14 @@ show_menu () {
            sed "s@#DIR#@${PWD}@g" config/craftbeerpiboot > /etc/init.d/craftbeerpiboot
            chmod 755 /etc/init.d/craftbeerpiboot;
 
-           whiptail --title "Installition Finished" --msgbox "CraftBeerPi installation finished! You must hit OK to continue." 8 78
+           whiptail --title "Installation Finished" --msgbox "CraftBeerPi installation finished! You must hit OK to continue." 8 78
            show_menu
            ;;
        2)
           confirmAnswer "Are you sure you want to clear the CraftBeerPi. All hardware setting will be deleted"
           if [ $? = 0 ]; then
             sudo rm -f craftbeerpi.db
-            whiptail --title "Database Delted" --msgbox "The CraftBeerPi database was succesfully deleted. You must hit OK to continue." 8 78
+            whiptail --title "Database Deleted" --msgbox "The CraftBeerPi database was successfully deleted. You must hit OK to continue." 8 78
             show_menu
           else
            show_menu
@@ -81,7 +82,7 @@ show_menu () {
              sed "s@#DIR#@${PWD}@g" config/craftbeerpiboot > /etc/init.d/craftbeerpiboot
              chmod 755 /etc/init.d/craftbeerpiboot;
              update-rc.d craftbeerpiboot defaults;
-             whiptail --title "Added succesfull to autostart" --msgbox "The CraftBeerPi was added to autostart succesfully. You must hit OK to continue." 8 78
+             whiptail --title "Added successful to autostart" --msgbox "The CraftBeerPi was added to autostart successfully. You must hit OK to continue." 8 78
              show_menu
            else
             show_menu
@@ -104,7 +105,7 @@ show_menu () {
            ;;
        6)
            sudo /etc/init.d/craftbeerpiboot stop
-           whiptail --title "CraftBeerPi stoped" --msgbox "The software is stoped" 8 78
+           whiptail --title "CraftBeerPi stopped" --msgbox "The software is stopped" 8 78
            show_menu
             ;;
        7)
@@ -118,7 +119,7 @@ show_menu () {
            fi
            ;;
         8)
-           confirmAnswer "Are you sure you want to reset all file changes for this git respository (git reset --hard)?"
+           confirmAnswer "Are you sure you want to reset all file changes for this git repository (git reset --hard)?"
            if [ $? = 0 ]; then
               whiptail --textbox /dev/stdin 20 50 <<<"$(git reset --hard)"
               show_menu
@@ -155,6 +156,17 @@ show_menu () {
               show_menu
             fi
             ;;
+		12)
+            confirmAnswer "Are you sure you want to install KairosDB?"
+            if [ $? = 0 ]; then
+			  wget https://github.com/kairosdb/kairosdb/releases/download/v1.2.1/kairosdb_1.2.1-1_all.deb
+              sudo dpkg -i kairosdb_1.2.1-1_all.deb
+			  sudo service kairosdb start
+			  show_menu
+            else
+              show_menu
+            fi
+            ;;
        esac
    fi
 }

modules/app_config.py

@@ -1,23 +1,16 @@
+from flask import Flask, json, g
+from flask_socketio import SocketIO
 
-import json
-import sys, os
-from flask import Flask, render_template, redirect, json, g
-
-
-from flask_socketio import SocketIO, emit
-
-import logging
-
-
+import yaml
+import logging.config
 
 from modules.core.core import CraftBeerPi, ActorBase, SensorBase
 from modules.core.db import DBModel
 
 app = Flask(__name__)
 
-FORMAT = '%(asctime)-15s - %(levelname)s - %(message)s'
+logging.config.dictConfig(yaml.load(open('./config/logger.yaml', 'r')))
 
-logging.basicConfig(filename='./logs/app.log',level=logging.INFO, format=FORMAT)
 app.config['SECRET_KEY'] = 'craftbeerpi'
 app.config['UPLOAD_FOLDER'] = './upload'
 

modules/core/core.py

@@ -1,16 +1,14 @@
 import inspect
-import pprint
 
-import sqlite3
 from flask import make_response, g
 import datetime
 from datetime import datetime
-from flask.views import MethodView
-from flask_classy import FlaskView, route
 
 from time import localtime, strftime
 from functools import wraps, update_wrapper
 
+import requests
+import json
 
 from props import *
 
@@ -18,6 +16,7 @@ from hardware import *
 
 import time
 import uuid
+import logging
 
 
 class NotificationAPI(object):
@@ -86,6 +85,8 @@ class SensorAPI(object):
         :return: 
         '''
 
+        self.logger = logging.getLogger(__name__)
+
         self.app.logger.info("Init Sensors")
 
         t = self.cache.get("sensor_types")
@@ -145,22 +146,48 @@ class SensorAPI(object):
 
     def receive_sensor_value(self, id, value):
         self.emit("SENSOR_UPDATE", self.cache.get("sensors")[id])
-        self.save_to_file(id, value)
+        self.save_to_file(id, value, "sensor")
+
+    def save_to_file(self, id, value, prefix):
+        sensor_name = "%s_%s" % (prefix, str(id))
+        use_kairosdb = (self.cache["config"]["kairos_db"].__dict__["value"] == "YES")
+
+        if use_kairosdb:
+            self.write_to_tsdb(prefix, sensor_name, value)
+        else:
+            self.write_to_logfile(sensor_name, value)
 
-    def save_to_file(self, id, value, prefix="sensor"):
-        filename = "./logs/%s_%s.log" % (prefix, str(id))
+    def write_to_logfile(self, sensor_name, value):
+        filename = "./logs/%s.log" % sensor_name
         formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime())
-        msg = str(formatted_time) + "," +str(value) + "\n"
+        msg = str(formatted_time) + "," + str(value) + "\n"
 
         with open(filename, "a") as file:
             file.write(msg)
 
+    def write_to_tsdb(self, prefix, sensor_name, value):
+        kairosdb_server = "http://127.0.0.1:" + self.cache["config"]["kairos_db_port"].__dict__["value"]
+
+        data = [
+            dict(name="cbpi." + sensor_name, datapoints=[
+                [int(round(time.time() * 1000)), value]
+            ], tags={
+                "cbpi": prefix,
+                "brew": self.cache["active_brew"]
+            })
+        ]
+
+        response = requests.post(kairosdb_server + "/api/v1/datapoints", json.dumps(data))
+        if not response.ok:
+            self.logger.warning("Failed to write time series entry for [%s]. Response [%s]", sensor_name, response)
+
     def log_action(self, text):
-        filename = "./logs/action.log"
-        formatted_time = strftime("%Y-%m-%d %H:%M:%S", localtime())
-        with open(filename, "a") as file:
-            text = text.encode("utf-8")
-            file.write("%s,%s\n" % (formatted_time, text))
+        use_kairosdb = (self.cache["config"]["kairos_db"].__dict__["value"] == "YES")
+
+        if use_kairosdb:
+            self.write_to_tsdb("action", "action", text)
+        else:
+            self.write_to_logfile("action", text)
 
     def shutdown_sensor(self, id):
         self.cache.get("sensors")[id].stop()
@@ -204,7 +231,8 @@ class CraftBeerPi(ActorAPI, SensorAPI):
         "messages": [],
         "plugins": {},
         "fermentation_controller_types": {},
-        "fermenter_task": {}
+        "fermenter_task": {},
+        "active_brew": "none"
     }
     buzzer = None
     eventbus = {}

modules/logs/endpoints.py

@@ -1,11 +1,15 @@
 import datetime
 import os
-from flask import Blueprint, request, send_from_directory, json
+import requests
+import logging
+from flask import request, send_from_directory, json
 from flask_classy import FlaskView, route
 from modules import cbpi
 
 
 class LogView(FlaskView):
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
 
     @route('/', methods=['GET'])
     def get_all_logfiles(self):
@@ -17,19 +21,10 @@ class LogView(FlaskView):
 
     @route('/actions')
     def actions(self):
-        filename = "./logs/action.log"
-        if os.path.isfile(filename) == False:
-            return
-        import csv
-        array = []
-        with open(filename, 'rb') as f:
-            reader = csv.reader(f)
-            for row in reader:
-                try:
-                    array.append([int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000, row[1]])
-                except:
-                    pass
-        return json.dumps(array)
+        array = self.read_log_as_json("action")
+        json_dumps = json.dumps(array)
+
+        return json_dumps
 
     @route('/<file>', methods=["DELETE"])
     def clearlog(self, file):
@@ -49,24 +44,85 @@ class LogView(FlaskView):
             cbpi.notify("Failed to delete log", "", type="danger")
         return ('', 204)
 
-    def read_log_as_json(self, type, id):
-        filename = "./logs/%s_%s.log" % (type, id)
-        if os.path.isfile(filename) == False:
-            return
+    def query_tsdb(self, sensor_name):
+        kairosdb_server = "http://127.0.0.1:" + cbpi.cache["config"]["kairos_db_port"].__dict__["value"]
+
+        data = dict(metrics=[
+            {
+                "tags": {},
+                "name": "cbpi.%s" % sensor_name,
+                "aggregators": [
+                    {
+                        "name": "avg",
+                        "align_sampling": True,
+                        "sampling": {
+                            "value": cbpi.cache["config"]["kairos_db_sampling_value"].__dict__["value"],
+                            "unit": "seconds"
+                        },
+                        "align_start_time": True
+                    }
+                ]
+            }
+        ],
+            cache_time=0,
+            start_relative={
+                "value": cbpi.cache["config"]["kairos_db_start_relative"].__dict__["value"],
+                "unit": "days"
+            })
+
+        if cbpi.cache["active_brew"] != "none":
+            data["metrics"][0]["tags"] = {"brew": [cbpi.cache["active_brew"]]}
+
+        self.logger.debug("query: %s", json.dumps(data))
+
+        response = requests.post(kairosdb_server + "/api/v1/datapoints/query", json.dumps(data))
+        if response.ok:
+            self.logger.debug("Fetching time series for [%s] took [%s]", sensor_name, response.elapsed)
+            self.logger.debug("Time series for [%s] is [%s]", sensor_name, response.json())
+            return response.json()["queries"][0]["results"][0]["values"]
+        else:
+            self.logger.warning("Failed to fetch time series for [%s]. Response [%s]", sensor_name, response)
 
-        import csv
+    def query_log(self, filename, value_type):
         array = []
+
+        if not os.path.isfile(filename):
+            self.logger.warn("File does not exist [%s]", filename)
+            return array
+
+        import csv
+
+        if value_type == "float":
+            converter = float
+        else:
+            converter = str
+
         with open(filename, 'rb') as f:
             reader = csv.reader(f)
             for row in reader:
                 try:
-                    array.append([int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S") - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000, float(row[1])])
+                    point_of_time = int((datetime.datetime.strptime(row[0], "%Y-%m-%d %H:%M:%S")
+                                         - datetime.datetime(1970, 1, 1)).total_seconds()) * 1000
+                    value = converter(row[1])
+                    array.append([point_of_time, value])
                 except:
+                    self.logger.exception("error in reading logfile [%s]", filename)
                     pass
         return array
 
+    def read_log_as_json(self, sensor_name):
+        use_kairosdb = (cbpi.cache["config"]["kairos_db"].__dict__["value"] == "YES")
+
+        if use_kairosdb:
+            return self.query_tsdb(sensor_name)
+        else:
+            filename = "./logs/%s.log" % sensor_name
+            return self.query_log(filename, "float")
+
+
     def convert_chart_data_to_json(self, chart_data):
-        return {"name": chart_data["name"], "data": self.read_log_as_json(chart_data["data_type"], chart_data["data_id"])}
+        return {"name": chart_data["name"],
+                "data": self.read_log_as_json(chart_data["data_type"] + "_" + str(chart_data["data_id"]))}
 
     @route('/<t>/<int:id>', methods=["POST"])
     def get_logs_as_json(self, t, id):
@@ -74,7 +130,8 @@ class LogView(FlaskView):
         result = []
         if t == "s":
             name = cbpi.cache.get("sensors").get(id).name
-            result.append({"name": name, "data": self.read_log_as_json("sensor", id)})
+            sensor_name = "%s_%s" % ("sensor", str(id))
+            result.append({"name": name, "data": self.read_log_as_json(sensor_name)})
 
         if t == "k":
             kettle = cbpi.cache.get("kettle").get(id)
@@ -82,7 +139,8 @@ class LogView(FlaskView):
 
         if t == "f":
             fermenter = cbpi.cache.get("fermenter").get(id)
-            result = map(self.convert_chart_data_to_json, cbpi.get_fermentation_controller(fermenter.logic).get("class").chart(fermenter))
+            result = map(self.convert_chart_data_to_json,
+                         cbpi.get_fermentation_controller(fermenter.logic).get("class").chart(fermenter))
 
         return json.dumps(result)
 
@@ -99,6 +157,7 @@ class LogView(FlaskView):
 
         return True if pattern.match(name) else False
 
+
 @cbpi.initalizer()
 def init(app):
     """

modules/recipe_import/beerxml.py

@@ -46,13 +46,13 @@ class BeerXMLImport(FlaskView):
     @route('/<int:id>', methods=['POST'])
     def load(self, id):
 
-
         steps = self.getSteps(id)
         boil_time_alerts = self.getBoilAlerts(id)
         name = self.getRecipeName(id)
         self.api.set_config_parameter("brew_name", name)
         boil_time = self.getBoilTime(id)
         mashstep_type = cbpi.get_config_parameter("step_mash", "MashStep")
+        mashinstep_type = cbpi.get_config_parameter("step_mashin", "MashInStep")
         mash_kettle = cbpi.get_config_parameter("step_mash_kettle", None)
 
         boilstep_type = cbpi.get_config_parameter("step_boil", "BoilStep")
@@ -65,10 +65,20 @@ class BeerXMLImport(FlaskView):
 
         try:
 
+            # Add mash in or mash step, depends on timer > 0
             for row in steps:
-                Step.insert(**{"name": row.get("name"), "type": mashstep_type, "config": {"kettle": mash_kettle, "temp": float(row.get("temp")), "timer": row.get("timer")}})
+                if row.get("timer") > 0:
+                    Step.insert(**{"name": row.get("name"), "type": mashstep_type,
+                                   "config": {"kettle": mash_kettle, "temp": float(row.get("temp")),
+                                              "timer": row.get("timer")}})
+                else:
+                    Step.insert(**{"name": row.get("name"), "type": mashinstep_type,
+                                   "config": {"kettle": mash_kettle, "temp": float(row.get("temp"))}})
+
+            # Add chilling step
             Step.insert(**{"name": "ChilStep", "type": "ChilStep", "config": {"timer": 15}})
-            ## Add boiling step
+
+            # Add boiling step
             Step.insert(**{
                 "name": "Boil",
                 "type": boilstep_type,
@@ -76,11 +86,11 @@ class BeerXMLImport(FlaskView):
                     "kettle": boil_kettle,
                     "temp": boil_temp,
                     "timer": boil_time,
-                    ## Beer XML defines additions as the total time spent in boiling,
-                    ## CBP defines it as time-until-alert
+                    # Beer XML defines additions as the total time spent in boiling,
+                    # CBP defines it as time-until-alert
 
-                    ## Also, The model supports five boil-time additions.
-                    ## Set the rest to None to signal them being absent
+                    # Also, The model supports five boil-time additions.
+                    # Set the rest to None to signal them being absent
                     "hop_1": boil_time - boil_time_alerts[0] if len(boil_time_alerts) >= 1 else None,
                     "hop_2": boil_time - boil_time_alerts[1] if len(boil_time_alerts) >= 2 else None,
                     "hop_3": boil_time - boil_time_alerts[2] if len(boil_time_alerts) >= 3 else None,
@@ -88,7 +98,8 @@ class BeerXMLImport(FlaskView):
                     "hop_5": boil_time - boil_time_alerts[4] if len(boil_time_alerts) >= 5 else None
                 }
             })
-            ## Add Whirlpool step
+
+            # Add Whirlpool step
             Step.insert(**{"name": "Whirlpool", "type": "ChilStep", "config": {"timer": 15}})
             StepView().reset()
             self.api.emit("UPDATE_ALL_STEPS", Step.get_all())
@@ -141,7 +152,12 @@ class BeerXMLImport(FlaskView):
             else:
                 temp = round(9.0 / 5.0 * float(e.find("STEP_TEMP").text) + 32, 2)
 
-            steps.append({"name": e.find("NAME").text, "temp": temp, "timer": float(e.find("STEP_TIME").text)})
+            if e.find("STEP_TIME").text is None:
+                stepTime = 0.0
+            else:
+                stepTime = float(e.find("STEP_TIME").text)
+
+            steps.append({"name": e.find("NAME").text, "temp": temp, "timer": stepTime})
 
         return steps
 

modules/steps/__init__.py

@@ -1,4 +1,5 @@
 import time
+import datetime
 from flask import json, request
 from flask_classy import route
 
@@ -93,6 +94,7 @@ class StepView(BaseView):
         self.model.reset_all_steps()
         self.stop_step()
         cbpi.emit("UPDATE_ALL_STEPS", self.model.get_all())
+        cbpi.cache["active_brew"] = "none"
         return ('', 204)
 
     def stop_step(self):
@@ -142,9 +144,15 @@ class StepView(BaseView):
         # set step instance to ache
         cbpi.cache["active_step"] = instance
 
-    @route('/next', methods=['POST'])
     @route('/start', methods=['POST'])
     def start(self):
+        if "none" == cbpi.cache["active_brew"]:
+            cbpi.cache["active_brew"] = cbpi.cache["config"]["brew_name"].__dict__["value"] + \
+                                    "_" + datetime.datetime.now().strftime('%y-%m-%dT%H:%M')
+        return self.next()
+
+    @route('/next', methods=['POST'])
+    def next(self):
         active = Step.get_by_state("A")
         inactive = Step.get_by_state('I')
 
@@ -163,6 +171,7 @@ class StepView(BaseView):
         else:
             cbpi.log_action("Brewing Finished")
             cbpi.notify("Brewing Finished", "You are done!", timeout=None)
+            cbpi.cache["active_brew"] = "none"
 
         cbpi.emit("UPDATE_ALL_STEPS", Step.get_all())
         return ('', 204)

modules/system/endpoints.py

@@ -61,6 +61,10 @@ class SystemView(FlaskView):
         for t in repo.tags:
             tags.append({"name": t.name, "commit": str(t.commit), "date": t.commit.committed_date,
                          "committer": t.commit.committer.name, "message": t.commit.message})
+        for b in repo.branches:
+            tags.append({"name": b.name, "commit": str(b.commit), "date": b.commit.committed_date,
+                         "committer": b.commit.committer.name, "message": b.commit.message})
+
         try:
             branch_name = repo.active_branch.name
             # test1

update/4_kairosdb_config.sql

@@ -0,0 +1,4 @@
+INSERT OR IGNORE INTO config VALUES ('kairos_db', 'NO', 'select', 'Use timeseries database KairosDB for storing sensor values. You can install KairosDB with the CraftBeerPi installer.', '["YES","NO"]' );
+INSERT OR IGNORE INTO config VALUES ('kairos_db_port', 8080, 'number', 'Port for KairosDB. We assume the DB is running on your PI, so IP-Address is 127.0.0.1.', NULL );
+INSERT OR IGNORE INTO config VALUES ('kairos_db_sampling_value', 5, 'number', 'A timeseries database has the advantage to aggregate data points and therefore to reduce the transmitted data. This value sets a time span in seconds to calculate the average', NULL );
+INSERT OR IGNORE INTO config VALUES ('kairos_db_start_relative', 1, 'number', 'If you have an ongoing brew or fermentation process only values related to this process will be shown in the graph. Additionally you can define in days how far the time series should reach in the past. The earliest time of both information defines when the data series starts.', NULL );