Develop Reference

ModuleNotFoundError When loading a pickled file in tornado

I am using tornado framework for loading my machine learning model. I have a popularity class
import numpy as np
import pandas as pd
from pandas import DataFrame
class Popularity():
users_data = pd.read_csv('~/Desktop/LatentCollaborativeFiltering/lib/seed_data/ratings.csv')
movies_data = pd.read_csv('~/Desktop/LatentCollaborativeFiltering/lib/seed_data/movies.csv')
data = pd.merge(users_data, movies_data, left_on="movieId", right_on="movieId")
data = pd.DataFrame.sort_values(data, ['userId','movieId'],ascending=[0,1])
def __init__(self):
pass
def favoriteMovies(self, activeUser,N):
topMovies=pd.DataFrame.sort_values(self.data[self.data.userId==activeUser],['rating'],ascending=[0])[:N]
# return the title corresponding to the movies in topMovies
return list(topMovies.title)
def recommend_movies(self):
return "No recommendation"
Now i have another file to pickle an object of this class build_model.py
from __future__ import print_function
import os
from sklearn.externals import joblib
import pandas as pd
import numpy as np
from popularity import Popularity
if __name__ == "__main__":
popu = Popularity()
_CUR_DIR = os.path.dirname(os.path.realpath(__file__))
_SERIALIZATION_DIR = os.path.join(_CUR_DIR)
if not os.path.exists(_SERIALIZATION_DIR):
os.makedirs(_SERIALIZATION_DIR)
model_filename = os.path.join(_SERIALIZATION_DIR, "model.pkl")
joblib.dump(popu, model_filename)
print("Successfully Built and Picked into models folder")
This now builds the model and successfully saves the model in same directory as model.pkl file. But when I load the model in the torando it gives me following error
[I 180702 06:30:44 server:40] Loading Latent Collaborative Filtering model...
Traceback (most recent call last):
File "run.py", line 7, in <module>
server.main()
File "/home/rabin/Desktop/LatentCollaborativeFiltering/movies-api/app/server.py", line 45, in main
MODELS["recommender"] = pickle.load(infile)
ModuleNotFoundError: No module named 'Popularity'
My server.py file is
# !/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import logging
import logging.config
import tornado.ioloop
import tornado.web
from tornado.options import options
from sklearn.externals import joblib
from app.settings import MODEL_DIR, ROOT_DIR, _CUR_DIR
from app.handler import IndexHandler, IrisPredictionHandler
from app.popularity import Popularity
import pickle
MODELS = {}
def load_model(pickle_filename):
return joblib.load(pickle_filename)
def main():
# Get the Port and Debug mode from command line options or default in settings.py
options.parse_command_line()
# create logger for app
logger = logging.getLogger('app')
logger.setLevel(logging.INFO)
FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=FORMAT)
# Load ML Models
logger.info("Loading Latent Collaborative Filtering model...")
#MODELS["recommender"] = load_model(os.path.join(_CUR_DIR, 'model.pkl'))
#MODELS["recommender"] = load_model('model.pkl')
with open(os.path.join(_CUR_DIR, 'model.pkl'), 'rb') as infile:
MODELS["recommender"] = pickle.load(infile)
urls = [
(r"/$", IndexHandler),
(r"/api/recommender/(?P<action>[a-zA-Z]+)?", RecommenderHandler, # action is function in handler
dict(model=MODELS["recommender"]))
]
# Create Tornado application
application = tornado.web.Application(
urls,
debug=options.debug,
autoreload=options.debug)
# Start Server
logger.info("Starting App on Port: {} with Debug Mode: {}".format(options.port, options.debug))
application.listen(options.port)
tornado.ioloop.IOLoop.current().start()
And my handler.py file is
"""
Request Handlers
"""
import tornado.web
from tornado import concurrent
from tornado import gen
from concurrent.futures import ThreadPoolExecutor
from app.base_handler import BaseApiHandler
from app.settings import MAX_MODEL_THREAD_POOL
from app.popularity import Popularity
class IndexHandler(tornado.web.RequestHandler):
"""APP is live"""
def get(self):
self.write("Movie Recommendation System is Live")
def head(self):
self.finish()
class RecommenderHandler(BaseApiHandler):
_thread_pool = ThreadPoolExecutor(max_workers=MAX_MODEL_THREAD_POOL)
def initialize(self, model, *args, **kwargs):
self.model = model
super().initialize(*args, **kwargs)
#concurrent.run_on_executor(executor='_thread_pool')
def _blocking_predict(self, X):
target_values = self.model.favoriteMovies(5,10)
return target_values
#gen.coroutine
def predict(self, data):
if type(data) == dict:
data = [data]
X = []
for item in data:
record = (item.get("user_id"))
X.append(record)
results = yield self._blocking_predict(X)
self.respond(results)
I have searched too much for the solution but has not found yet that worked for me.
I cannot load from the console too

Issue with unit test for tiny Flask app

I have a very primitive Flask application which works as I'm expecting, but I'm failing to write a unit test for it. The code of the app is following (i omit insignificant part):
app.py
from flask import *
import random
import string
app = Flask(__name__)
keys = []
app.testing = True
#app.route('/keygen/api/keys', methods=['POST'])
def create():
symbol = string.ascii_letters + string.digits
key = ''.join(random.choice(symbol) for _ in range(4))
key_instance = {'key': key, 'is_used': False}
keys.append(key_instance)
return jsonify({'keys': keys}), 201
The test is:
tests.py
import unittest
from flask import *
import app
class TestCase(unittest.TestCase):
def test_number_one(self):
test_app = Flask(app)
with test_app.test_client() as client:
rv = client.post('/keygen/api/keys')
...something...
if __name__ == '__main__':
unittest.main()
The traceback:
ERROR: test_number_one (__main__.TestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
File "tests.py", line 12, in test_number_one
test_app = Flask(app)
File "/Users/bulrathi/Yandex.Disk.localized/Virtualenvs/ailove/lib/python3.5/site-packages/flask/app.py", line 346, in __init__
root_path=root_path)
File "/Users/bulrathi/Yandex.Disk.localized/Virtualenvs/ailove/lib/python3.5/site-packages/flask/helpers.py", line 807, in __init__
root_path = get_root_path(self.import_name)
File "/Users/bulrathi/Yandex.Disk.localized/Virtualenvs/ailove/lib/python3.5/site-packages/flask/helpers.py", line 668, in get_root_path
filepath = loader.get_filename(import_name)
File "<frozen importlib._bootstrap_external>", line 384, in _check_name_wrapper
ImportError: loader for app cannot handle <module 'app' from '/Users/bulrathi/Yandex.Disk.localized/Обучение/Code/Django practice/ailove/keygen/app.py'>
----------------------------------------------------------------------
Ran 1 test in 0.003s
FAILED (errors=1)
Thanks for your time.

You have a few issues with the posted code (indentation aside):
First, in tests.py you import app and use it, but app is the module rather than the app object from app.py. You should import the actual app object using
from app import app
and secondly, you are using that app object (assuming we fix the import) as a parameter to another Flask() constructor—essentially saying:
app = Flask(Flask(app))
Since we've imported app from app.py, we can just use it directly, so we remove the app = Flask(app) line (and the associated import statement as we don't need it anymore) and your test file becomes:
import unittest
from app import app
class TestCase(unittest.TestCase):
def test_number_one(self):
with app.test_client() as client:
rv = client.post('/keygen/api/keys')
...something...
if __name__ == '__main__':
unittest.main()
You should note also that the from flask import * form is discouraged in favour of importing specific parts of the module, so
from flask import Flask, jsonify
would be better in your app.py.

How can I use mock for testing inside greenlet?

I use bottle & gevent for my python (2.7.6) application.
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from gevent import spawn, monkey
from bottle import Bottle
from .settings import MONGODB_HOST, MONGODB_PORT, MONGODB_NAME
monkey.patch_all()
mongo_client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = mongo_client[MONGODB_NAME]
class MyApp(object):
def insert_event(self):
data = {'a': self.a, 'b': self.b} # some data
db.events.insert(data)
def request(self):
# request data processing...
spawn(self.insert_event)
return {}
app = Bottle()
app.route('/', method='POST')(MyApp().request)
And I want to test it with mongomock (https://github.com/vmalloc/mongomock).
from __future__ import unicode_literals
from unittest import TestCase
from webtest import TestApp
from mock import patch
from mongomock import MongoClient
from ..app import app as my_app
db = MongoClient().db
#patch('my_app.app.db', db)
class TestViews(TestCase):
def setUp(self):
self.app = TestApp(ssp_app)
self.db = db
def test_request(self):
response = self.app.post('/', {})
last_event = self.db.events.find_one({})
self.assertTrue(last_event)
My test fails.
FAIL: test_request (my_app.tests.TestViews)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/mock/mock.py", line 1305, in patched
return func(*args, **keywargs)
File "/srv/mysite/my_app/tests/views.py", line 71, in test_request
self.assertTrue(last_event)
AssertionError: None is not true
It is work if I use self.insert_event without spawn. I tried to use patch.object, "with" statement, but without success...

I found solution. I need to mock gevent.spawn method. Because I get HTTP response before the coroutine ends. This my solution:
#patch('my_app.app.db', db)
#patch('my_app.app.spawn',
lambda method, *args, **kwargs: method(*args, **kwargs))
class TestViews(TestCase):

ImportError during unittest script

I'm trying to set some tests for my server.
According to a friend's recommendation I wanna use unittest and mock. For this purpose I wrote some lines but when I try to execute the script I got an importError.
Traceback (most recent call last):
File "test_creds.py", line 146, in <module>
unittest.main()
AttributeError: 'module' object has no attribute 'main'
Do I have some mistake with the imports?
Thanks!
# coding=utf-8
import sys, os
import unittest, mock, kiss
from twisted.python import log
from twisted.trial import unittest
from twisted.cred import credentials
class CredentialsChecker(unittest.TestCase):
"""
Testing the server credentials handling
"""
def _setUp_databases(self):
username_1 = 'user_name'
password_1 = 'user_pass'
email_1 = 'user#mail.org'
create_user_profile = mock.Mock()
self.db = mock.Mock()
self.db.username = username_1
self.db.password = password_1
self.db.email = email_1
def setUp(self):
log.startLogging(sys.stdout)
log.msg(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Flushing database")
#management.execute_from_command_line(['manage.py', 'flush',\
#'--noinput'])
log.msg(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Populating database")
#management.execute_from_command_line(['manage.py', 'createsuperuser',\
#'--username', 'superuser_name', '--email', 'superuser_name#email.org',\
#'--noinput'])
self._setUp_databases()
log.msg(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Running tests")
return
"""
Log in with valid credentianls. The server should return True
"""
def test_GoodCredentials(self):
creds = credentials.UsernamePassword('user_name', 'user_pass')
checker = DjangoAuthChecker()
d = checker.requestAvatarId(creds)
if __name__ == '__main__':
unittest.main()

If you create an object inheriting from the class unittest.TestCase test you have to use the native Python unittest module.

Error "ImportError: Loaded module flask.testing not found in sys.modules" while testing flask app in GAE?

I am using nose and nose-gae for testing flask app in App engine. I using virtualenv.
Test case looks like this:
import os
import unittest
from google.appengine.ext import testbed
from tickapp import app
class DemoTest(unittest.TestCase):
def setUp(self):
self.app = app.test_client()
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_all_stubs()
def teardown(self):
self.testbed.deactivate()
def test_home_page(self):
result = self.app.get('/')
self.assertEqual(result.status, "200 OK")
if __name__ == "__main__":
main()
Running tests using: nosetests -v --with-gae tests
Test case files reside under 'tests' directory.
Stacktrace:
Traceback (most recent call last):
File "/home/raj/gae_projects/tick/tests/test_users.py", line 13, in setUp
self.app = app.test_client()
File "/home/raj/gae_projects/tick/flask/app.py", line 781, in test_client
from flask.testing import FlaskClient as cls
ImportError: Loaded module flask.testing not found in sys.modules
Thanks for any help.