I have 2 modules as:
main.py
get.py
In main.py, I need to call a function present in get.py which is call_get().
So I'm trying it this way:
import get
get.call_get()
But it throws an error as 'Attribute Error'.
On the other hand, the following code works:
import temp
temp.func()
function func in temp.py looks as:
import get
get.call_get()
I am unable to understand this behaviour. Please guide.
get.py code:
import requests
import json
import sys
import utils
import error_handler
import get_helper
import os
import pvdata
def call_get():
try:
auth_tuple = utils.get_auth_details("get")
headers = utils.get_header()
resource_types = utils.get_resource_types()
namespaces = utils.get_namespaces()
if resource_types[0].lower() == "all":
resource_types = utils.append_all_resources()
get_helper.get_namespace_list(auth_tuple, headers)
all_namespaces = utils.extract_namespaces_from_list("source")
if namespaces[0].lower() != "all":
error_handler.validate_source_namespaces(namespaces, all_namespaces)
utils.create_file(
"Namespaces", "All_namespaces_at_source.txt", str(all_namespaces))
get_helper.generate_json_for_all_namespaces(
all_namespaces, auth_tuple, headers)
for resource_name in resource_types:
if namespaces[0].lower() == "all":
for namespace in all_namespaces:
get_helper.call_all_functions_for_get(
namespace, resource_name, headers, auth_tuple)
else:
for namespace in namespaces:
get_helper.call_all_functions_for_get(
namespace, resource_name, headers, auth_tuple)
except Exception as error:
filename = os.path.basename(__file__)
error_handler.print_exception_message(error, filename)
return
if __name__ == "__main__":
call_get()
main.py code:
import utils
import remote_exec
import post
import get
import error_handler
import os
import handle_space
import socket
import json
from requests import get
import sys
import temp
def only_dest_requires_jumpserver():
try:
dictionary = {
"migration_type": utils.config_data()["source_cloud"] + " to " + utils.config_data()["dest_cloud"]
}
utils.update_config_file(dictionary)
print("\nInitialising " + utils.config_data()["source_cloud"] + " to " + utils.config_data()["dest_cloud"] + " migration...")
hostname = socket.gethostname()
if hostname == utils.config_data()["my_hostname"]:
# get.call_get()
temp.func()
print("\nData successfully exported from source to this machine.\nChecking for space availability at jumpserver...")
print("Done!")
except Exception as error:
filename = os.path.basename(__file__)
error_handler.print_exception_message(error, filename)
The issue is main.py has 2 get modules as:
import get
from requests import get
get is being overwritten.....you need to rename your function or use
import request
request.get
Another simple way is aliasing suggested by InsertCheesyLine.
from requests import get as _get
and use _get
Related
Objective : To Create UnitTestCases for main.py
How can we import another python file which will dynamically return result
File : main.py
import os
_ENV = os.popen("echo ${RPM_ENVIRONMENT}").read().split('\n')[0]
_HOST = os.popen("echo $(hostname)").read().split('\n')[0]
_JOB_TYPE=os.popen("echo ${JOB_TYPE}").read().split('\n')[0]
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Import another python file to our testing script
when i import main.py , i will receive error on SERVER_URL = { KeyError '${RPM_ENVIRONMENT}'
I believe the only reason why its returning error is because it does not recognize RPM_ENVIRONMENT, how can we mock _ENV variables in our main file and print server url as https://dev.net
After i have succesfully import main.py in my test case file, then i will create my unitTest cases as the ENV variable is required for me.
Testing : test_main.py
import unittest, sys
from unittest import mock
# --> Error
sys.path.insert(1, 'C:/home/src')
import main.py as conf
# For an example : we should be able to print https://dev.net when we include this in our line
URL = conf.SERVER_URL.get('ENV_URL')
ENV = conf._ENV
#URL should return https://dev.net & ENV should return DEV
class test_tbrp_case(unittest.TestCase):
def test_port(self):
#function yet to be created
pass
if __name__=='__main__':
unittest.main()
There's little reason to shell out of Python. You can read an environment variable with os.environ. os.environ['RPM_ENVIRONMENT'].
import os
_ENV = os.environ['RPM_ENVIRONMENT']
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Now your test can set RPM_ENVIRONMENT before importing main.py.
os.environ['RPM_ENVIRONMENT'] = 'UAT'
sys.path.insert(1, 'C:/home/src')
import main.py as conf
I have a data in a website(which unfortunately couldn't be fetched using invokeHTTP). I am trying to fetch the data using a execute script processor.Here is what I have done in python.tried things in comments
#import http.client, urllib.request, urllib.parse, urllib.error, base64
import httplib
#import requests
import java.io
from org.apache.commons.io import IOUtils
from java.nio.charset import StandardCharsets
from org.apache.nifi.processor.io import OutputStreamCallback
import json
from datetime import datetime
from java.lang import Object
from jarray import array
from org.python.core.util import StringUtil
class WriteContentCallback(OutputStreamCallback):
def __init__(self, content):
self.content_text = content
def process(self, outputStream):
try:
outputStream.write(StringUtil.toBytes(self.content_text))
except Exception as error:
objArray = [error]
javaArray = array(objArray, Object)
log.error('Error processing the .csv file ===> {}', javaArray)
raise ValueError
try:
flowFile = session.create()
conn = httplib.HTTPSConnection(url_domain)
conn.request("GET", url)
response = conn.getresponse()
data = response.read()
#outputStream.write(StringUtil.toBytes(data))
#outputStream.write(bytearray(json.dumps(out)))
flowFile = session.write(flowFile, WriteContentCallback(data))
session.transfer(flowFile, REL_SUCCESS)
except Exception as outermost_error:
objArray = [outermost_error]
javaArray = array(objArray, Object)
log.error('Error processing the .csv file ===> {}', javaArray)
session.transfer(flowFile, REL_FAILURE)
The issue seems to be there are no errors as such...but the file doesnt show up(0 bytes) and the type is application/octetstream.What can I do to get the output from the website using execute script
Here is the nifi flow, the flow has been marked in yellow.
Let me know if you need any other details. the code seems to work as jython program.And also if i pass some random text in place of data,that works too.
Python unable to import package, but works correctly from within the package. A fully functional example below. In the virtual env I am using 3.6 All responses greatly appreciated!
parsers/
__init__.py
element.py
parser1.py
parser2.py
parserresolver.py
outsidepkg.py
init.py is empty
element.py:
def router():
pass
parser1.py:
from element import *
def parse(data):
return data
parser2.py:
from element import *
def parse(data):
return data
parserresolver.py:
import sys
from parser1 import *
from parser2 import *
def resolve(data):
parseddata = None
parsers = ['parser1', 'parser2']
funcname = 'parse'
for parser in parsers:
module = sys.modules[parser]
if hasattr(module, funcname):
func = getattr(module, funcname)
parseddata = func(data)
print(parseddata)
return parseddata
if __name__ == "__main__":
resolve('something')
outsidepkg.py:
import parsers.parserresolver
def getapi(data):
parsers.parserresolver.resolve(data)
if __name__ == "__main__":
print(getapi('in parse api main'))
So when I call parserresolver.py directly it works great, no import errors and prints out "something" as expected.
But when I call outsidepkg.py I am getting this error:
Traceback (most recent call last):
File "C:\code\TestImport\TestImport\outsidepkg.py", line 1, in <module>
import parsers.parserresolver
File "C:\code\TestImport\TestImport\parsers\parserresolver.py", line 2, in <module>
from parser1 import *
ModuleNotFoundError: No module named 'parser1'
Press any key to continue . . .
You need to change the imports of:
from file import whatever
To:
from .file import whatever
Since your code to run it is outside the folder, use a . to get the directory, since the file isn't outside the package.
I have a Python file named pythontc.py which consists of the below code:
import unittest
import genson
import requests,json
from requests.auth import HTTPBasicAuth
from jsonschema import validate
class TC1(tu.Tc):
def setUp(self):
with open('schema.json', 'q') as t:
self.schema = t.read()
self.file = t
def test1(self):
result =requests.get('http://localhost:8000/some/123/somename?attribute=somename&someid=123&provider=somename&refresh=true', auth=HTTPBasicAuth('USERNAME', 'Pass'))
data = result.text
print data
def tearDown(self):
self.file.close()
if __name__ == '__main__':
tu.main()
Now this exexutes in cmd now i want to get that result in html format using htmltest runner.
I used the command in cmd as:
cmd>HTMLTestRunner.py pythontc >result.html
But i dont want to type manually every time so what can i add in python code ?
I'm writing a simple script that:
Loads a big list of URLs
Get the content of each URL making concurrent HTTP requests using requests' async module
Parses the content of the page with lxml in order to check if a link is in the page
If the link is present on the page, saves some info about the page in a ZODB database
When I test the script with 4 or 5 URLs It works well, I only have the following message when the script ends:
Exception KeyError: KeyError(45989520,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
But when I try to check about 24000 URLs it fails toward the end of the list (when there are about 400 URLs left to check) with the following error:
Traceback (most recent call last):
File "check.py", line 95, in <module>
File "/home/alex/code/.virtualenvs/linka/local/lib/python2.7/site-packages/requests/async.py", line 83, in map
File "/home/alex/code/.virtualenvs/linka/local/lib/python2.7/site-packages/gevent-1.0b2-py2.7-linux-x86_64.egg/gevent/greenlet.py", line 405, in joinall
ImportError: No module named queue
Exception KeyError: KeyError(45989520,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
I tried both with the version of gevent available on pypi and downloading and installing the latest version (1.0b2) from gevent repository.
I cannot understand why this happened, and why it happened only when I check a bunch of URLs. Any suggestions?
Here is the entire script:
from requests import async, defaults
from lxml import html
from urlparse import urlsplit
from gevent import monkey
from BeautifulSoup import UnicodeDammit
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
import transaction
import persistent
import random
storage = FileStorage('Data.fs')
db = DB(storage)
connection = db.open()
root = connection.root()
monkey.patch_all()
defaults.defaults['base_headers']['User-Agent'] = "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20100101 Firefox/11.0"
defaults.defaults['max_retries'] = 10
def save_data(source, target, anchor):
root[source] = persistent.mapping.PersistentMapping(dict(target=target, anchor=anchor))
transaction.commit()
def decode_html(html_string):
converted = UnicodeDammit(html_string, isHTML=True)
if not converted.unicode:
raise UnicodeDecodeError(
"Failed to detect encoding, tried [%s]",
', '.join(converted.triedEncodings))
# print converted.originalEncoding
return converted.unicode
def find_link(html_doc, url):
decoded = decode_html(html_doc)
doc = html.document_fromstring(decoded.encode('utf-8'))
for element, attribute, link, pos in doc.iterlinks():
if attribute == "href" and link.startswith('http'):
netloc = urlsplit(link).netloc
if "example.org" in netloc:
return (url, link, element.text_content().strip())
else:
return False
def check(response):
if response.status_code == 200:
html_doc = response.content
result = find_link(html_doc, response.url)
if result:
source, target, anchor = result
# print "Source: %s" % source
# print "Target: %s" % target
# print "Anchor: %s" % anchor
# print
save_data(source, target, anchor)
global todo
todo = todo -1
print todo
def load_urls(fname):
with open(fname) as fh:
urls = set([url.strip() for url in fh.readlines()])
urls = list(urls)
random.shuffle(urls)
return urls
if __name__ == "__main__":
urls = load_urls('urls.txt')
rs = []
todo = len(urls)
print "Ready to analyze %s pages" % len(urls)
for url in urls:
rs.append(async.get(url, hooks=dict(response=check), timeout=10.0))
responses = async.map(rs, size=100)
print "DONE."
I'm not sure what's the source of your problem, but why do you have monkey.patch_all() not at the top of the file?
Could you try putting
from gevent import monkey; monkey.patch_all()
at the top of your main program and see if it fixes anything?
I am such a big n00b but anyway, I can try ... !
I guess you can try to change your import list by this one :
from requests import async, defaults
import requests
from lxml import html
from urlparse import urlsplit
from gevent import monkey
import gevent
from BeautifulSoup import UnicodeDammit
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
import transaction
import persistent
import random
Try this and tell me if it works .. I guess that can solve your problem :)
good day.
I think it's open python bug with number Issue1596321
http://bugs.python.org/issue1596321