I am using Python's unittest library to add a unit test. This how the folder structure looks like in my project.
Project
|-- __init__.py
|
|-- src
| |
| |-- __init__.py
| |-- main.py
|
|-- test
|-- __init__.py
|-- test_basic.py
Code for main.py
from dotenv import load_dotenv
load_dotenv()
import newrelic.agent
newrelic.agent.initialize()
from .processAnalyticsData import run_query_and_upload_results_to_s3, upload_script_results_to_s3
import sys
import click
import getopt
import json
import os
BUCKET_NAME = os.environ["S3_BUCKET_NAME"]
cwd = os.getcwd()
#click.group(chain=True)
def run_scripts():
pass
#run_scripts.command('run_script')
#click.option('-f', '--frequency', required=True, nargs=1, type=click.Choice(['WEEKLY', 'DAILY', 'ONCE', 'MONTHLY'], case_sensitive=False))
def run_script(frequency):
with open(cwd + '/src/config.json') as config_file:
script_paths = json.load(config_file)
output_json = [x for x in script_paths["executable_scripts"] if x["frequency"] == frequency]
for item in output_json:
file_name = item["fileName"]
script_name = item["path"]
script_type = item["type"]
if script_type == 'sql':
run_query_and_upload_results_to_s3(script_name, BUCKET_NAME, file_name)
elif script_type == 'python':
upload_script_results_to_s3(script_name, BUCKET_NAME, file_name)
else:
raise Exception('Script type is incorrect. Please provide the correct value in the config file.')
if __name__ == "__main__":
run_scripts()
I started writing my unit test like this:
from src.main import *
import unittest
class WidgetTestCase(unittest.TestCase):
def setUp(self):
pass
if __name__ == '__main__':
unittest.main()
I get an error AttributeError: 'module' object has no attribute 'test_basic'
However, If I remove the line from src.main import * and run the test using python -m test.test_basic , it works perfectly fine. Is there a problem with the imports ?
Upon running python -c "from src.main import *"
I noticed this error
File "<string>", line 1, in <module>
File "src/main.py", line 1, in <module>
from dotenv import load_dotenv
ImportError: No module named dotenv ```
Related
Objective : To Create UnitTestCases for main.py
How can we import another python file which will dynamically return result
File : main.py
import os
_ENV = os.popen("echo ${RPM_ENVIRONMENT}").read().split('\n')[0]
_HOST = os.popen("echo $(hostname)").read().split('\n')[0]
_JOB_TYPE=os.popen("echo ${JOB_TYPE}").read().split('\n')[0]
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Import another python file to our testing script
when i import main.py , i will receive error on SERVER_URL = { KeyError '${RPM_ENVIRONMENT}'
I believe the only reason why its returning error is because it does not recognize RPM_ENVIRONMENT, how can we mock _ENV variables in our main file and print server url as https://dev.net
After i have succesfully import main.py in my test case file, then i will create my unitTest cases as the ENV variable is required for me.
Testing : test_main.py
import unittest, sys
from unittest import mock
# --> Error
sys.path.insert(1, 'C:/home/src')
import main.py as conf
# For an example : we should be able to print https://dev.net when we include this in our line
URL = conf.SERVER_URL.get('ENV_URL')
ENV = conf._ENV
#URL should return https://dev.net & ENV should return DEV
class test_tbrp_case(unittest.TestCase):
def test_port(self):
#function yet to be created
pass
if __name__=='__main__':
unittest.main()
There's little reason to shell out of Python. You can read an environment variable with os.environ. os.environ['RPM_ENVIRONMENT'].
import os
_ENV = os.environ['RPM_ENVIRONMENT']
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Now your test can set RPM_ENVIRONMENT before importing main.py.
os.environ['RPM_ENVIRONMENT'] = 'UAT'
sys.path.insert(1, 'C:/home/src')
import main.py as conf
I have a Flask application that is constructed as following:
--- /app
------ /__pycache__
------ /static
--------- /files
--------- /assets
--------- /images
------ /templates
------ api.py
------ utils.py
------ config.py
------ sftp.py
--- wsgi.py
wsgi.py:
from app.main import app
if __name__ == "__main__":
app.run()
main.py:
from flask import Flask, render_template, request, session, flash, url_for, redirect
from werkzeug.utils import secure_filename
from .utils import *
from .config import *
from .sftp import sync_work_days
app = Flask(__name__)
app.secret_key = b'very_secret_key'
[...]
config.py:
import os
from wsgi import app
ROOT_PATH = app.instance_path
sftp.py:
import pysftp as sftp
from .config import *
from .utils import upload_working_days
import time, os
from datetime import datetime
from config import ROOT_PATH
def sync_work_days():
cnopts = sftp.CnOpts()
cnopts.hostkeys = None
s = sftp.Connection(host=SFTP_HOSTNAME, username=SFTP_USERNAME, password=SFTP_PASSWORD, cnopts=cnopts)
print("Connection succesfully established...")
remoteFilePath = '/files/company'
files = s.listdir_attr(remoteFilePath)
if files == []:
return "empty"
else:
for f in files:
localFilePath = os.path.join(ROOT_PATH, 'static/files/company')
# s.get(remoteFilePath, localFilePath)
upload_working_days(localFilePath)
time.sleep(5)
print("UPLOAD DONE")
s.rename(remoteFilePath + f, remoteFilePath + "/archives/" + f)
When I run the app, I get the following error:
from wsgi import app
ImportError: cannot import name 'app' from partially initialized module 'wsgi' (most likely due to a circular import) (D:\[...redacted...]\wsgi.py)
All I want to do is to download a file via SSH and put it in app/static/files, and I can't just hardcode the path because the deployment environment is different.
UPDATE 1:
I tried to cut off config.py from the loop and I made the following changes:
sftp.py:
import pysftp as sftp
from .config import *
from .main import app
from .utils import upload_working_days
import time, os
from datetime import datetime
def sync_work_days():
cnopts = sftp.CnOpts()
cnopts.hostkeys = None
filename = os.path.join(app.instance_path, 'my_folder', 'my_file.txt')
And this is the error I get:
ImportError: cannot import name 'app' from partially initialized module 'app.main' (most likely due to a circular import) (D:\[...redacted...]\app\main.py)
You have a circular import because you are importing app in config.py from wsgi.py, which gets it from main.py which imports config.py.
And you are importing app in config.py because you need a setting from it. Have you considered keeping all your settings in config.py and importing them in main.py from there? That seems like a logical thing to do.
The solution to this problem is the following:
In sftp.py, replace:
from .main import app
by:
from . import main
and instead of using app.instance_path, use main.app.instance_path
List below is my project structure.
├── folder1
├── test.py
├── folder2
├── A.py
├── folder3
├── A.py
A.py in folder2 and folder3 are the same except self.key
#folder2
class TestAPI():
def __init__(self):
self.key = 50
#folder3
class TestAPI():
def __init__(self):
self.key = 100
In test.py, the code is
import sys
root_path = '/user/my_account'
sys.path.insert(0, root_path + '/folder2')
from A import TestAPI
test_api = TestAPI()
print(test_api.key)
sys.path.insert(0, root_path + '/folder3')
from A import TestAPI
test_api = TestAPI()
print(test_api.key)
print(sys.path)
While executing test.py, it returns 50,50, ['/user/my_account/folder3', '/user/my_account/folder2']. Why the second time from A import TestAPI not from folder3 but folder2?
Edit: If I'd like to import TestAPI from folder3 for the second time, is there a way to 'delete' the PYTHONPATH after import TestAPI from folder2?
I would change your TestAPI code (A.py) into a commons folder (one that can be imported by other codes) and then have:
class TestAPI():
def __init__(self, key):
self.key = key
Then in your test.py you can do something like:
import sys
root_path = '/user/my_account'
sys.path.insert(0, root_path + '/commons')
from commons.A import TestAPI
test_api_50 = TestAPI(50)
print(test_api_50.key)
test_api_100 = TestAPI(100)
print(test_api_100.key)
I can edit this if there's an issue/typo.
Note that python folder/packaging is worth reading about. Your future self will thank you.
importlib.reload solve my question.
import sys
root_path = '/user/my_account'
sys.path.insert(0, root_path + '/folder2')
import A
test_api = A.TestAPI()
print(test_api.key)
sys.path.insert(0, root_path + '/folder3')
import importlib
importlib.reload(A) # should reload after inserting PYTHONPATH and before import A
import A
test_api = A.TestAPI()
print(test_api.key)
print(sys.path)
sys.path is a list. So just use the append method.
Also, you don't need root path. just run
sys.path.append('folder2')
Edit: demo that prooves that it works
[luca#artix stackoverflow]$ mkdir my_module
[luca#artix stackoverflow]$ ed my_module/foo.py
?my_module/foo.py
i
print('From foo')
.
wq
18
[luca#artix stackoverflow]$ python
Python 3.9.1 (default, Feb 6 2021, 13:49:29)
[GCC 10.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import foo
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ModuleNotFoundError: No module named 'foo'
>>> import sys
>>> sys.path.append('my_module')
>>> import foo
From foo
>>>
Following is my project structure:
root directory
| - __init__.py
| - notdoneyet.py
| - helpers.py
| - opencv_generators.py
| - seam_carve.py
| - imgtovideos.py
notdoneyet.py file contains the entry point of project and the remaining scripts are imported as modules and when required.
My __init__.py contains following code:
from .notdoneyet import user_input
from .helpers import createFolder
from .helpers import getFileExtension
from .helpers import writeImage
from .opencv_generators import generateEnergyMap
from .opencv_generators import generateColorMap
from .imagetovideos import generateVideo
from .imagetovideos import getToProcessPaths
from .seam_carve import cropByColumn
from .seam_carve import cropByRow
I have published the package on testPyPI. But when I try to import it after installing on local machine, I get the import error.
Initial code for notdoneyet.py:
import os, sys, cv2, argparse
#Local imports
import imgtovideos as itv
import helpers as hp #Error on this line
import opencv_generators as og
import seam_carve as sc
def main(argsip):
#usr inpt
I am getting the error "no module named helpers"
Here is screenshot of the error:
Please help me.
Thank you.
__init__.py are use to mark directories on disk as a Python package directories. lets say we have the files
py/src/__init__.py
py/src/user.py
and py is on your path, you can import the code in user.py as:
import src.module
or
from src import user
If we remove the __init__.py file, Python will no longer look for submodules inside that directory, so attempts to import the module will fail.
But on my app the __init__.py is not working.
__init__ .py
__all__ = ['models', 'common']
from src.models.user import User
from src.models.post import Post
from src.models.blog import Blog
When I run python app.py
errors:
Traceback (most recent call last):
File "src/app.py", line 5, in <module>
from src.models.user import User
ModuleNotFoundError: No module named 'src'
App.py file:
import os
import sys
from flask import Flask, render_template, request, session
from src.models.user import User
app = Flask(__name__) #'__main__'
#app.route('/') # 'OUrsite.com/api/'
# def hello world ot test the first api call
def hello_method():
return render_template('login.html')
#app.route('/login')
def login_user():
email = request.form['email']
password = request.form['password']
if User.login_valid(email, password):
User.login(email)
return render_template("profile.html", email=session['email'])
if __name__ == '__main__':
app.run(port = 9100)
Am I missing something?
There are two things to understand.
There is a difference in root directory of virtualenv and your system environment.
Virtualenv sets your project dir to it's root. But in case you are not using virtualenv, the root would be your system directory (at least, in my experience).
So, while creating __init__py, keep in mind that, if there is no virtualenv or you haven't imported anything, it can be empty.
simple __init_.py examples:
# in your __init__.py
from file import File
# now import File from package
from package import File
My __init__.py:
import src
from src import *
from .models.user import User
from .models.post import Post
from .models.blog import Blog
And when you import in app.py or src/models/user.py which is in a same directory & subdirectory, you shall not include src:
from models.user import User
from common.database import Database
You didn't import src first.
import src
__all__ = ['models', 'common']
from src.models.user import User
from src.models.post import Post
from src.models.blog import Blog
If this is src/__init__.py, you can use a relative import:
from __future__ import absolute_import
from .models.user import User
from .models.post import Post
from .models.blog import Blog