From 95c7027239918a35dad52650076cbfcfa8c2680f Mon Sep 17 00:00:00 2001 From: esensar Date: Thu, 20 Sep 2018 22:05:03 +0200 Subject: [PATCH] Add redis and celery support --- .env | 3 +++ Procfile | 3 ++- app/__init__.py | 46 -------------------------------------- app/accounts/__init__.py | 2 +- app/accounts/models.py | 2 +- app/celery_builder.py | 9 ++++++++ app/core.py | 48 ++++++++++++++++++++++++++++++++++++++++ app/dashboard/models.py | 2 +- app/devices/__init__.py | 2 +- app/devices/models.py | 2 +- app/mqtt/mqtt_client.py | 4 ++++ app/tasks/__init__.py | 0 app/tasks/celery.py | 18 +++++++++++++++ app/tasks/mqtt_tasks.py | 0 config.py | 5 +++++ requirements.txt | 8 ++++++- 16 files changed, 101 insertions(+), 53 deletions(-) create mode 100644 .env create mode 100644 app/celery_builder.py create mode 100644 app/core.py create mode 100644 app/tasks/__init__.py create mode 100644 app/tasks/celery.py create mode 100644 app/tasks/mqtt_tasks.py diff --git a/.env b/.env new file mode 100644 index 0000000..30a3321 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +DATABASE_URL="postgresql://localhost" +REDIS_URL=redis://localhost:6379 +CELERY_TASK_SERIALIZER=json diff --git a/Procfile b/Procfile index 0f0e082..f7dd9da 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ release: ./release-tasks.sh -web: gunicorn app:app -w 4 --preload +web: gunicorn app.core:app -w 4 --preload +worker: celery -A app.celery_builder.celery worker diff --git a/app/__init__.py b/app/__init__.py index 4d744fc..e69de29 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,46 +0,0 @@ -# App initialization -from flask_api import FlaskAPI -from flask_sqlalchemy import SQLAlchemy -from flask_bcrypt import Bcrypt -from flasgger import Swagger -from flask_cors import CORS - -app = FlaskAPI(__name__, instance_relative_config=True) -app.config.from_object('config') -app.config.from_pyfile('config.py', silent=True) -db = SQLAlchemy(app) -bcrypt = Bcrypt(app) -swagger = Swagger(app, template_file='swagger/template.yaml') -CORS(app) - - -def setup_blueprints(app): - """ - Sets up all of the blueprints for application - - All blueprints should be imported in this method and then added - - API blueprint should expose all resources, while other - blueprints expose other domain specific functionalities - They are exposed as blueprints just for consistency, otherwise - they are just simple python packages/modules - """ - from .accounts import accounts_bp - from .devices import devices_bp - from .dashboard import dashboard_bp - from .api import api_bp - from .mqtt import mqtt_bp - - app.register_blueprint(accounts_bp) - app.register_blueprint(devices_bp) - app.register_blueprint(dashboard_bp) - app.register_blueprint(mqtt_bp) - app.register_blueprint(api_bp, url_prefix='/api') - - -setup_blueprints(app) - - -@app.route("/") -def root(): - return "Hello World!" diff --git a/app/accounts/__init__.py b/app/accounts/__init__.py index 655e6c8..fdf6aaa 100644 --- a/app/accounts/__init__.py +++ b/app/accounts/__init__.py @@ -1,4 +1,4 @@ -from app import bcrypt +from app.core import bcrypt from flask import Blueprint from .models import Account diff --git a/app/accounts/models.py b/app/accounts/models.py index f366615..eeedcc6 100644 --- a/app/accounts/models.py +++ b/app/accounts/models.py @@ -1,6 +1,6 @@ import jwt import datetime -from app import db, app +from app.core import db, app from calendar import timegm diff --git a/app/celery_builder.py b/app/celery_builder.py new file mode 100644 index 0000000..c3126e0 --- /dev/null +++ b/app/celery_builder.py @@ -0,0 +1,9 @@ +# App initialization +from flask import Flask +from .tasks import celery as celery_configurator + +app = Flask(__name__, instance_relative_config=True) +app.config.from_object('config') +app.config.from_pyfile('config.py', silent=True) +app.config['MQTT_CLIENT_ID'] = 'final-iot-backend-server-worker' +celery = celery_configurator.make_celery(app) diff --git a/app/core.py b/app/core.py new file mode 100644 index 0000000..ef5a5bc --- /dev/null +++ b/app/core.py @@ -0,0 +1,48 @@ +# App initialization +from flask_api import FlaskAPI +from flask_sqlalchemy import SQLAlchemy +from flask_bcrypt import Bcrypt +from flasgger import Swagger +from flask_cors import CORS +from .tasks import celery as celery_configurator + +app = FlaskAPI(__name__, instance_relative_config=True) +app.config.from_object('config') +app.config.from_pyfile('config.py', silent=True) +db = SQLAlchemy(app) +bcrypt = Bcrypt(app) +swagger = Swagger(app, template_file='swagger/template.yaml') +CORS(app) +celery = celery_configurator.make_celery(app) + + +def setup_blueprints(app): + """ + Sets up all of the blueprints for application + + All blueprints should be imported in this method and then added + + API blueprint should expose all resources, while other + blueprints expose other domain specific functionalities + They are exposed as blueprints just for consistency, otherwise + they are just simple python packages/modules + """ + from .accounts import accounts_bp + from .devices import devices_bp + from .dashboard import dashboard_bp + from .api import api_bp + from .mqtt import mqtt_bp + + app.register_blueprint(accounts_bp) + app.register_blueprint(devices_bp) + app.register_blueprint(dashboard_bp) + app.register_blueprint(mqtt_bp) + app.register_blueprint(api_bp, url_prefix='/api') + + +setup_blueprints(app) + + +@app.route("/") +def root(): + return "Hello World!" diff --git a/app/dashboard/models.py b/app/dashboard/models.py index fc7beb3..b389105 100644 --- a/app/dashboard/models.py +++ b/app/dashboard/models.py @@ -1,4 +1,4 @@ -from app import db +from app.core import db from sqlalchemy.dialects.postgresql import JSON diff --git a/app/devices/__init__.py b/app/devices/__init__.py index 9704fbb..6f76473 100644 --- a/app/devices/__init__.py +++ b/app/devices/__init__.py @@ -1,7 +1,7 @@ import sys from flask import Blueprint from .models import Device, Recording, DeviceAssociation, DeviceType -from app import app +from app.core import app devices_bp = Blueprint('devices', __name__) diff --git a/app/devices/models.py b/app/devices/models.py index bed15a9..5489cf7 100644 --- a/app/devices/models.py +++ b/app/devices/models.py @@ -1,5 +1,5 @@ from datetime import datetime -from app import db +from app.core import db from sqlalchemy.dialects.postgresql import JSON diff --git a/app/mqtt/mqtt_client.py b/app/mqtt/mqtt_client.py index cc5c408..c5c7000 100644 --- a/app/mqtt/mqtt_client.py +++ b/app/mqtt/mqtt_client.py @@ -27,6 +27,10 @@ class MqttClient: def handle_disconnect(): print('MQTT client disconnected') + @MqttClient.mqtt.on_log() + def handle_logging(client, userdata, level, buf): + print(level, buf) + print('MQTT client initialized') @staticmethod diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/tasks/celery.py b/app/tasks/celery.py new file mode 100644 index 0000000..5c19bbd --- /dev/null +++ b/app/tasks/celery.py @@ -0,0 +1,18 @@ +from celery import Celery + + +def make_celery(app): + celery = Celery( + app.import_name, + backend=app.config['CELERY_RESULT_BACKEND'], + broker=app.config['CELERY_BROKER_URL'] + ) + celery.conf.update(app.config) + + class ContextTask(celery.Task): + def __call__(self, *args, **kwargs): + with app.app_context(): + return self.run(*args, **kwargs) + + celery.Task = ContextTask + return celery diff --git a/app/tasks/mqtt_tasks.py b/app/tasks/mqtt_tasks.py new file mode 100644 index 0000000..e69de29 diff --git a/config.py b/config.py index 3dc450e..f9e0237 100644 --- a/config.py +++ b/config.py @@ -28,12 +28,17 @@ CSRF_SESSION_KEY = "secret" SECRET_KEY = "?['Z(Z\x83Y \x06T\x12\x96<\xff\x12\xe0\x1b\xd1J\xe0\xd9ld" # MQTT configuration +MQTT_CLIENT_ID = 'final-iot-backend-server' MQTT_BROKER_URL = 'broker.hivemq.com' MQTT_BROKER_PORT = 1883 MQTT_USERNAME = 'user' MQTT_PASSWORD = 'secret' MQTT_REFRESH_TIME = 1.0 # refresh time in seconds +# Celery config +CELERY_BROKER_URL = os.environ['REDIS_URL'] +CELERY_RESULT_BACKEND = os.environ['REDIS_URL'] + # Flassger config SWAGGER = { 'uiversion': 3 diff --git a/requirements.txt b/requirements.txt index f47c021..892b0f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ alembic==0.9.9 +amqp==2.3.2 aniso8601==3.0.0 bcrypt==3.1.4 +billiard==3.5.0.4 +celery==4.2.0 cffi==1.11.5 click==6.7 flasgger==0.8.3 @@ -17,6 +20,7 @@ gunicorn==19.8.1 itsdangerous==0.24 Jinja2==2.10 jsonschema==2.6.0 +kombu==4.2.1 Mako==1.0.7 MarkupSafe==1.0 marshmallow==3.0.0b9 @@ -29,9 +33,11 @@ PyJWT==1.6.1 python-dateutil==2.7.2 python-editor==1.0.3 pytz==2018.4 -PyYAML==3.12 +PyYAML==3.13 +redis==2.10.6 six==1.11.0 SQLAlchemy==1.2.7 typing==3.6.4 +vine==1.1.4 webargs==3.0.0 Werkzeug==0.14.1