Develop Reference

Filtering python logging module to only log last line of an exception?

I'm utilizing the following code to print exceptions from a python module I'm running inside of my script:
except Exception as e:
logging.info('yt-dlp initated a break', exc_info=True)
Currently it outputs much more information than I need.
Something along the lines of:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/web_ripper.py", line 58, in subscriptions_loop
ydl.download([url])
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 3113, in download
self.__download_wrapper(self.extract_info)(
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 3086, in wrapper
res = func(*args, **kwargs)
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 1344, in extract_info
return self.__extract_info(url, self.get_info_extractor(ie_key), download, extra_info, process)
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 1371, in wrapper
self.report_error(str(e), e.format_traceback())
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 912, in report_error
self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args, **kwargs)
File "/app/venv/lib/python3.9/site-packages/yt_dlp/YoutubeDL.py", line 853, in trouble
raise DownloadError(message, exc_info)
yt_dlp.utils.DownloadError: ERROR: [youtube:tab] Videogamedunkey: Unable to download API page: HTTP Error 404: Not Found
What's the most efficient way to simply print the last line of the exception?

You can use a custom Formatter that will process the exc_text of the record you are trying to log.
The following MyFormatter will check if the record has an exception text. If not, it will create it using self.formatException. Then, since record.exc_text is a string storing the trace as text, with \n as separators between lines, you can keep the last line using .split("\n")[-1].
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
# https://github.com/python/cpython/blob/main/Lib/logging/__init__.py#L711
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
record.exc_text = record.exc_text.split("\n")[-1]
return super().format(record)
Then, add the following code to use your new MyFormatter:
def a(x):
return b(x)
def b(x):
return c(x)
def c(x):
return d(x)
def d(x):
return x / 0
logger = logging.getLogger("foobar")
logger.setLevel(logging.INFO)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(MyFormatter(fmt="%(name)s - %(levelname)s - %(message)s"))
logger.addHandler(stream_handler)
try:
a(1)
except Exception as e:
logger.info('yt-dlp initated a break', exc_info=True)
and you get:
foobar - INFO - yt-dlp initated a break
ZeroDivisionError: division by zero
instead of the previous:
foobar - INFO - yt-dlp initated a break
Traceback (most recent call last):
File "70704112.py", line 31, in <module>
a(1)
File "70704112.py", line 12, in a
return b(x)
File "70704112.py", line 15, in b
return c(x)
File "70704112.py", line 18, in c
return d(x)
File "70704112.py", line 21, in d
return x / 0
ZeroDivisionError: division by zero

ResourceNotFoundError: Operation returned an invalid status 'Not Found' when using the get function for Azure virtual machines?

When trying to get information about a virtual machine, I am met with the following error:
Traceback (most recent call last):
File "/Users/shilpakancharla/Documents/function_app/WeedsMediaUploadTrigger/event_process.py", line 73, in <module>
vm_data, vm_status = get_azure_vm(key.RESOURCE_GROUP, key.VIRTUAL_MACHINE_NAME)
File "<decorator-gen-2>", line 2, in get_azure_vm
File "/usr/local/lib/python3.9/site-packages/retry/api.py", line 73, in retry_decorator
return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter,
File "/usr/local/lib/python3.9/site-packages/retry/api.py", line 33, in __retry_internal
return f()
File "/Users/shilpakancharla/Documents/function_app/WeedsMediaUploadTrigger/event_process.py", line 65, in get_azure_vm
vm_instance = compute_client.virtual_machines.get(resource_group_name,
File "/usr/local/lib/python3.9/site-packages/azure/mgmt/compute/v2019_12_01/operations/_virtual_machines_operations.py", line 641, in get
map_error(status_code=response.status_code, response=response, error_map=error_map)
File "/usr/local/lib/python3.9/site-packages/azure/core/exceptions.py", line 102, in map_error
raise error
azure.core.exceptions.ResourceNotFoundError: Operation returned an invalid status 'Not Found'
Does this mean that get() is not a valid function for compute_client.virtual_machines? It says that it is valid in the documentation, but I am not sure why this error is arising. This is my implementation, and I use the call vm_data, vm_status = get_azure_vm(key.RESOURCE_GROUP, key.VIRTUAL_MACHINE_NAME). I am confident that all my credentials are correct.
def get_access_to_virtual_machine():
subscription_id = key.SUBSCRIPTION_ID
credentials = DefaultAzureCredential(authority = AzureAuthorityHosts.AZURE_GOVERNMENT,
exclude_environment_credential = True,
exclude_managed_identity_credential = True,
exclude_shared_token_cache_credential = True)
client = KeyClient(vault_url = key.VAULT_URL, credential = credentials)
compute_client = ComputeManagementClient(credentials, subscription_id,
base_url = 'https://portal.azure.us')
return compute_client
def get_azure_vm(resource_group_name, virtual_machine_name):
compute_client = get_access_to_virtual_machine()
vm_instance = compute_client.virtual_machines.get(resource_group_name,
virtual_machine_name)
vm_status = vm_instance.instance_view.statuses[1].display_status
return vm_instance, vm_status

You should specify expand , if you want to get a VM instance View . Try the code below :
compute_client.virtual_machines.get(resource_group_name, virtual_machine_name,expand='instanceView').instance_view.statuses[1].display_status
Result:

How to handle "not found" ticker in yfinance?

So im trying to fetch some stock data in a loop (not sure if i can pass an array), like this:
def getData(ticker):
print (ticker)
data = pdr.get_data_yahoo(ticker, start=start_date, end=today)
dataname= ticker+'_'+str(today)
files.append(dataname)
SaveData(data, dataname)
But for some reasons, some of the tickers i feed to pdr.get_data_yahoo() are not found, and python throws this error:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/pandas_datareader/yahoo/daily.py", line 157, in _read_one_data
data = j["context"]["dispatcher"]["stores"]["HistoricalPriceStore"]
KeyError: 'HistoricalPriceStore'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "borsdata_api.py", line 65, in <module>
getData(row['ticker'])
File "borsdata_api.py", line 47, in getData
data = pdr.get_data_yahoo(ticker, start=start_date, end=today)
File "/usr/local/lib/python3.7/site-packages/pandas_datareader/data.py", line 82, in get_data_yahoo
return YahooDailyReader(*args, **kwargs).read()
File "/usr/local/lib/python3.7/site-packages/pandas_datareader/base.py", line 251, in read
df = self._read_one_data(self.url, params=self._get_params(self.symbols))
File "/usr/local/lib/python3.7/site-packages/pandas_datareader/yahoo/daily.py", line 160, in _read_one_data
raise RemoteDataError(msg.format(symbol, self.__class__.__name__))
pandas_datareader._utils.RemoteDataError: No data fetched for symbol ADDV-TO-1.ST using YahooDailyReader
Is it possible to just skip this iteration and move on the next one in the list?

def getData(ticker):
print (ticker)
try:
data = pdr.get_data_yahoo(ticker, start=start_date, end=today)
dataname= ticker+'_'+str(today)
files.append(dataname)
SaveData(data, dataname)
except:
pass #or traceback.print_exc(), or traceback.format_exc()
#print_exc() will raise the error and print traceback.
#format_exc() will return the error as a string.

Python3.6: File "/usr/lib/python3.6/configparser.py", line 1138, in _unify_values

My script is working fine on Pycharm with python 3.6, but has some issues when i run from ubuntu 16 using Python3.6
Error:
python3.6 AttributeInsertion.py
Loading library modules...
Traceback (most recent call last):
File "/usr/lib/python3.6/configparser.py", line 1138, in _unify_values
sectiondict = self._sections[section]
KeyError: 'test_configurations'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "AttributeInsertion.py", line 49, in <module>
jsonFeatureFile = readstringconfigfile('test_configurations', 'resourceName')
File "AttributeInsertion.py", line 15, in readstringconfigfile
fieldvalue = parser.get(section, field)
File "/usr/lib/python3.6/configparser.py", line 781, in get
d = self._unify_values(section, vars)
File "/usr/lib/python3.6/configparser.py", line 1141, in _unify_values
raise NoSectionError(section)
configparser.NoSectionError: No section: 'test_configurations'
Error in sys.excepthook:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 63, in apport_excepthook
from apport.fileutils import likely_packaged, get_recent_crashes
File "/usr/lib/python3/dist-packages/apport/__init__.py", line 5, in <module>
from apport.report import Report
File "/usr/lib/python3/dist-packages/apport/report.py", line 30, in <module>
import apport.fileutils
File "/usr/lib/python3/dist-packages/apport/fileutils.py", line 23, in <module>
from apport.packaging_impl import impl as packaging
File "/usr/lib/python3/dist-packages/apport/packaging_impl.py", line 23, in <module>
import apt
File "/usr/lib/python3/dist-packages/apt/__init__.py", line 23, in <module>
import apt_pkg
ModuleNotFoundError: No module named 'apt_pkg'
Original exception was:
Traceback (most recent call last):
File "/usr/lib/python3.6/configparser.py", line 1138, in _unify_values
sectiondict = self._sections[section]
KeyError: 'test_configurations'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "AttributeInsertion.py", line 49, in <module>
jsonFeatureFile = readstringconfigfile('test_configurations', 'resourceName')
File "AttributeInsertion.py", line 15, in readstringconfigfile
fieldvalue = parser.get(section, field)
File "/usr/lib/python3.6/configparser.py", line 781, in get
d = self._unify_values(section, vars)
File "/usr/lib/python3.6/configparser.py", line 1141, in _unify_values
raise NoSectionError(section)
configparser.NoSectionError: No section: 'test_configurations'
from configparser import ConfigParser
from time import sleep
import json
import datetime
import requests
print('Loading library modules...')
def readstringconfigfile(section, field):
try:
parser = ConfigParser()
configfilename = "..\\resources\\config.ini"
parser.read(configfilename)
fieldvalue = parser.get(section, field)
print(f'Read string config file {fieldvalue}.')
return fieldvalue[1:-1]
except FileNotFoundError:
print("Cannot find the config file.")
def replace_property_value(property, value):
try:
print('Old ' + property + ': ' + feature['properties'][property]) # print old value
feature['properties'][property] = value # override old value with new
print('New ' + property + ': ' + feature['properties'][property])
except Exception as e:
print(repr(e))
def replace_payload_value(value):
try:
print('Old payload: ' + feature['properties']['payload'][0])
feature['properties']['payload'][0] = value
print('New payload: ' + feature['properties']['payload'][0])
except Exception as e:
print(repr(e))
def replace_user_name(value):
try:
print(feature['properties']['user']['name'])
feature['properties']['user']['name'] = value
print('New payload: ' + feature['properties']['user']['name'])
except Exception as e:
print(repr(e))
jsonFeatureFile = readstringconfigfile('test_configurations', 'resourceName')
jsonFeatureFile = "{0}{1}".format("..\\resources\\", jsonFeatureFile)
print(f'Resource file name is {jsonFeatureFile}.')
xyzSitBearerToken = readstringconfigfile('sit_configurations', 'xyzSitBearerToken')
xyzSitBearerToken = "\"" + xyzSitBearerToken + "\""
print(f'XYZ Bearer Token is {xyzSitBearerToken}.')
# Read FeatureCollection from JSON file:
print('Reading FeatureCollection from JSON file.')
with open(jsonFeatureFile, 'r', encoding='utf-8') as f:
try:
featureJson = json.load(f)
print('Opening FeatureCollection.json', f)
#print(f'Raw feature from resource file is {json.dumps(rawFeatureJson, indent=4)}.')
except Exception as e:
print(repr(e))
try:
headers = {'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(xyzSitBearerToken)}
e2eXyzSemiPerm = readstringconfigfile('sit_configurations', 'e2eXyzSemiPerm')
e2eXyzBaseUrl = readstringconfigfile('sit_configurations', 'e2eXyzBaseUrl')
xyzPutUrl = e2eXyzBaseUrl + e2eXyzSemiPerm + "/features"
print(f'PUT request to XYZ url - {xyzPutUrl}.')
for feature in featureJson['attributes']:
currentTime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%SZ')
print(f'Current Time is {currentTime}.')
replace_property_value("creationTime", currentTime)
replace_property_value("startTime", currentTime)
replace_property_value("informationType", "gottlieb-sit-semi-permanent")
replace_payload_value("0xFF")
replace_user_name("Pradeep Thomas Thundiyil")
print(f'PUT Feature - {json.dumps(feature, indent=4)}.')
response = requests.put(xyzPutUrl, json=feature, headers=headers)
print(f'PUT feature to XYZ url - {response.url}.')
print(f'PUT request response to XYZ {response.text}.')
print(f'Response of PUT request to XYZ {response.status_code}.')
print(response.json)
print(f'Sleeping for 20 seconds.')
sleep(20)
except KeyError as e:
print(repr(e))

Here the path provided in the variable 'configfilename' is not pointing towards the exact location of config.ini file.
Default directories can be different according to the environments/servers/modules/etc; which is not the directory in which your program exists. So, you can make the code point towards the current directory where the file exists.
You can try by importing OS module(import os) and following changes in your variable 'configfilename' which directs to 'config.ini' file under the directory 'resources'.
configfilename = (os.path.join(os.path.dirname(__file__),'resources','config.ini'))

How to get a complete exception stack trace in Python

The following snippet:
import traceback
def a():
b()
def b():
try:
c()
except:
traceback.print_exc()
def c():
assert False
a()
Produces this output:
Traceback (most recent call last):
File "test.py", line 8, in b
c()
File "test.py", line 13, in c
assert False
AssertionError
What should I use if I want the complete stack trace including the call to a?
If it matters I have Python 2.6.6
edit: What I'd like to get is the same information I'd get if I left the try except out and let the exception propagate to the top level. This snippet for example:
def a():
b()
def b():
c()
def c():
assert False
a()
Produces this output:
Traceback (most recent call last):
File "test.py", line 10, in <module>
a()
File "test.py", line 2, in a
b()
File "test.py", line 5, in b
c()
File "test.py", line 8, in c
assert False
AssertionError

Here's a function based on this answer. It will also work when no exception is present:
def full_stack():
import traceback, sys
exc = sys.exc_info()[0]
stack = traceback.extract_stack()[:-1] # last one would be full_stack()
if exc is not None: # i.e. an exception is present
del stack[-1] # remove call of full_stack, the printed exception
# will contain the caught exception caller instead
trc = 'Traceback (most recent call last):\n'
stackstr = trc + ''.join(traceback.format_list(stack))
if exc is not None:
stackstr += ' ' + traceback.format_exc().lstrip(trc)
return stackstr
print full_stack() will print the full stack trace up to the top, including e.g. IPython's interactiveshell.py calls, since there is (to my knowledge) no way of knowing who would catch exceptions. It's probably not worth figuring out anyway...
If print full_stack() is called from within an except block, full_stack will include the stack trace down to the raise. In the standard Python interpreter, this will be identical to the message you receive when not catching the exception (Which is why that del stack[-1] is there, you don't care about the except block but about the try block).

I don't know if there is a better way, but here's what I did:
import traceback
import sys
def format_exception(e):
exception_list = traceback.format_stack()
exception_list = exception_list[:-2]
exception_list.extend(traceback.format_tb(sys.exc_info()[2]))
exception_list.extend(traceback.format_exception_only(sys.exc_info()[0], sys.exc_info()[1]))
exception_str = "Traceback (most recent call last):\n"
exception_str += "".join(exception_list)
# Removing the last \n
exception_str = exception_str[:-1]
return exception_str
def main1():
main2()
def main2():
try:
main3()
except Exception as e:
print "Printing only the traceback above the current stack frame"
print "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
print
print "Printing the full traceback as if we had not caught it here..."
print format_exception(e)
def main3():
raise Exception()
if __name__ == '__main__':
main1()
And here's the output I get:
Printing only the traceback above the current stack frame
Traceback (most recent call last):
File "exc.py", line 22, in main2
main3()
File "exc.py", line 31, in main3
raise Exception()
Exception
Printing the full traceback as if we had not caught it here...
Traceback (most recent call last):
File "exc.py", line 34, in <module>
main1()
File "exc.py", line 18, in main1
main2()
File "exc.py", line 22, in main2
main3()
File "exc.py", line 31, in main3
raise Exception()
Exception

Use
traceback.print_stack()
http://docs.python.org/library/traceback.html#traceback.print_stack
suxmac2 $ python out.py
File "out.py", line 16, in <module>
a()
File "out.py", line 5, in a
b()
File "out.py", line 11, in b
traceback.print_stack()

Here is a bit better variant of Tobias Kienzler answer. It works same, but can be called not right in except block, but somewhere deeper.
In other words, this variant will print same stacks, when called like
try:
...
except Exception:
print full_stack()
or
def print_full_stack():
print full_stack()
try:
...
except Exception:
print_full_stack()
Here is code:
def full_stack():
import traceback, sys
exc = sys.exc_info()[0]
if exc is not None:
f = sys.exc_info()[-1].tb_frame.f_back
stack = traceback.extract_stack(f)
else:
stack = traceback.extract_stack()[:-1] # last one would be full_stack()
trc = 'Traceback (most recent call last):\n'
stackstr = trc + ''.join(traceback.format_list(stack))
if exc is not None:
stackstr += ' ' + traceback.format_exc().lstrip(trc)
return stackstr