I am kind of new to Python and playing around with classes. I created a class to access aws ec2 instances to stop and start along with current state. Don't know how else to explain this other than showing the issue.
#!/usr/bin/python
import boto3
import json
import datetime
from botocore.exceptions import ClientError
class Ec2(object):
def __init__(self, instance_id, region):
self.instances = [instance_id]
self.region = 'us-gov-west-1'
self.state = ''
if region is not None:
self.region = region
self.ec2 = boto3.client('ec2', region_name=region)
self._setState()
def _setState(self):
def convert_timestamp(datetime_key):
if isinstance(datetime_key, (datetime.date, datetime.datetime)):
return datetime_key.timestamp()
status = self.ec2.describe_instances(InstanceIds=self.instances)
# Find the current status of the instance
dump = json.dumps(status, default=convert_timestamp)
body = json.loads(dump)
self.state = body["Reservations"][0]["Instances"][0]["State"]["Name"]
def getState(self):
self._setState()
return self.state
...
So if in the Python3 interpretor I run this:
from ec2 import Ec2
ssm_1 = Ec2('i-12345abcdef','us-gov-west-1')
ssm_1.getState()
Python 3.8.0 (default, Oct 28 2019, 16:14:01)
[GCC 8.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from ec2 import Ec2
>>> ssm_1 = Ec2('i-12345abcdef','us-gov-west-1')
>>> ssm_1.getState()
'stopped'
>>>
If I change the method _setState in the class to:
def _setState(self):
def convert_timestamp(datetime_key):
if isinstance(datetime_key, (datetime.date, datetime.datetime)):
return datetime_key.timestamp()
status = self.ec2.describe_instances(InstanceIds=self.instances)
# Find the current status of the instance
dump = json.dumps(status, default=convert_timestamp)
body = json.loads(dump)
self.state = body["Reservations"][0]["Instances"][0]["State"]["Name"]
self.state = self.state.replace("'", "")
then rerun above the single quotes do not get removed! It appears that these quotes are not part of the actual string, validated by changing the last line to:
self.state = self.state[1:-1]
>>> ssm_1.getState()
'toppe'
so it removed the first and last letters not the single quotes, s & d
BUT if I change _setState to:
def _setState(self):
def convert_timestamp(datetime_key):
if isinstance(datetime_key, (datetime.date, datetime.datetime)):
return datetime_key.timestamp()
status = self.ec2.describe_instances(InstanceIds=self.instances)
# Find the current status of the instance
dump = json.dumps(status, default=convert_timestamp)
body = json.loads(dump)
self.state = body["Reservations"][0]["Instances"][0]["State"]["Name"]
Now run:
>>> from ec2 import Ec2
>>> ssm_1 = Ec2('i-12345abcdef','us-gov-west-1')
>>> ssm_1.getState().replace("'", "")
stopped
>>>
the single quotes are now gone!
So it appears in the class the string does not actually include the single quotes but outside the class it does. Why? Thanks in advance.
Related
I have been using Python to run a piece of software via a plug-in which is implemented as a DLL. This is provided through the following wrapper class:
from ctypes import *
import os
ANULL = -999999
gwbdll = None
if os.name == 'posix':
gwbdll = cdll.LoadLibrary('libgwbplugin.so')
else:
gwbdll = cdll.LoadLibrary('gwbplugin')
class GWBplugin:
Name = "GWBplugin"
def __init__(self):
self.plugin = c_void_p (None)
def initialize (self,app_name,file_name=None,cmds=None):
return gwbdll.c_initialize(byref(self.plugin), c_char_p(app_name.encode()), file_name if file_name == None else c_char_p(file_name.encode()), cmds if cmds == None else c_char_p(cmds.encode()))
def exec_cmd (self,uline):
return gwbdll.c_exec_cmd(byref(self.plugin), c_char_p(uline.encode()))
def results (self, value, units=None, ix=0, jy=0):
type = c_void_p
type_result = gwbdll.c_results(byref(self.plugin),c_void_p(None),c_char_p(value.encode()),c_char_p("DATATYPE".encode()),c_int(ix),c_int(jy))
if type_result == 1:
type = c_int
elif type_result == 2:
type = c_double
elif type_result == 3:
type = c_char_p
else:
return []
count = gwbdll.c_results(byref(self.plugin),c_void_p(None),c_char_p(value.encode()),units if units == None else c_char_p(units.encode()),c_int(ix),c_int(jy))
arr = (type*count)()
gwbdll.c_results(byref(self.plugin),cast(arr,c_void_p),c_char_p(value.encode()),units if units == None else c_char_p(units.encode()),c_int(ix),c_int(jy))
if type == c_char_p:
arr = [x.decode('cp1252') for x in arr]
return arr
def destroy (self):
gwbdll.c_destroy(byref(self.plugin))
def __del__(self):
gwbdll.c_destroy(byref(self.plugin))
I am running my program on a Jupyter notebook with Python 3.8.8 with code that looks like:
myPlugin = GWBplugin()
myPlugin.initialize("react", f_out, f_in_flagged)
where "react" is the name of the specific application I am using in this software, f_out looks like 'Output Files/pH 9/Reduced/0.0% Reduced/Output_6.txt', and f_in_flagged looks like "-i 'Input Files/pH 9/Reduced/0.0% Reduced/Input_6.rea'".
This is contained in a loop that runs through many different input files, and was running just fine until a few days ago when I generated more input files (contained within some new subdirectories) to run, and now it spits out the following error:
OSError Traceback (most recent call last)
<ipython-input-6-fdf290a73be1> in <module>
24 # #Initialize: Application, Output file, Input file containing setup
---> 25 myPlugin.initialize("react", f_out, f_in_flagged)
26 #Run
27 myPlugin.exec_cmd("go")
C:\Program Files\Gwb\src\GWBplugin.py in initialize(self, app_name, file_name, cmds)
15
16 def initialize (self,app_name,file_name=None,cmds=None):
---> 17 return gwbdll.c_initialize(byref(self.plugin), c_char_p(app_name.encode()), file_name if file_name == None else c_char_p(file_name.encode()), cmds if cmds == None else c_char_p(cmds.encode()))
18
19 def exec_cmd (self,uline):
OSError: exception: access violation reading 0x0000000000000000
I am not really familiar with C or ctypes, but as far as I can tell this has something to do with the name of the input and output files I'm feeding it. I tried going back and running the files in the original directories that were working before (even tried completely uninstalling and reinstalling everything: the software, anaconda, and deleted all of the new files and directories) and am now getting the same error, so beyond that I am really at a loss as to what is going on. Any help is greatly appreciated!
I am running Python 3.9 with Confluent Kafka 1.7.0, avro-python3 1.10.0 and fastavro 1.4.1.
The following code uses Avro schema encoder in order to encode a message, which succeeds only if we transform the resulting schema encoding by getting rid of the MappingProxyType:
from confluent_kafka import Producer
from confluent_kafka.avro import CachedSchemaRegistryClient, MessageSerializer
from fastavro.schema import parse_schema
from fastavro.validation import validate
from types import MappingProxyType
from typing import Any
import sys
def transformMap(item: Any) -> Any:
if type(item) in {dict, MappingProxyType}:
return {k:transformMap(v) for k,v in item.items()}
elif type(item) is list:
return [transformMap(v) for v in item]
else:
return item
def main(argv = None):
msgType = 'InstrumentIdMsg'
idFigi = 'BBG123456789'
head = {'sDateTime': 1, 'msgType': msgType, 'srcSeq': 1,
'rDateTime': 1, 'src': 'Brownstone', 'reqID': None,
'sequence': 1}
msgMap = {'head': head, 'product': 'Port', 'idIsin': None, 'idFigi': idFigi,
'idBB': None, 'benchmark': None, 'idCusip': None,'idCins': None}
registryClient = CachedSchemaRegistryClient(url = 'http://local.KafkaRegistry.com:8081')
schemaId, schema, version = registryClient.get_latest_schema(msgType)
serializer = MessageSerializer(registry_client = registryClient)
schemaMap = schema.to_json()
# NOTE:
# schemaMap cannot be used since it uses mappingproxy
# which causes validate() and parse_schema() to throw
schemaDict = transformMap(schemaMap)
isValid = validate(datum = msgMap, schema = schemaDict, raise_errors = True)
parsed_schema = parse_schema(schema = schemaDict)
msg = serializer.encode_record_with_schema_id(schema_id = schemaId,
record = msgMap)
producer = Producer({'bootstrap.servers': 'kafkaServer:9092'})
producer.produce(key = idFigi,
topic = 'TOPIC_NAME',
value = msg)
return 0
if __name__ == '__main__':
sys.exit(main())
The transformation basically leaves everything unchanged except altering MappingProxyType to dict instances.
Is there a problem in the way I am calling the standard library which causes mapping proxy to be used, which in turn causes fastavro to throw? Can this be fixed by something as a user, or is this really a bug in the Confluent Kafka library?
In addition, the output schemaId from registryClient.get_latest_schema() is marked in the docs to return str but returns int. If I understand correctly, this is the intended input into the schema_id parameter of serializer.encode_record_with_schema_id() (and it works correctly if I call it), which is also marked as int. Is that a typo in the docs? In other words, it seems either registryClient.get_latest_schema() should return an integer, or serializer.encode_record_with_schema_id() should take a string, or I am doing something incorrectly :) Which one is it?
Thank you very much.
I have an python application that collects data from an MQTT broker and presents it to a website via the gviz python API:
DESCRIPTION= [ \
('Zeit', 'datetime'), \
('Temperatur', 'number'), \
('Feuchtigkeit', 'number'),\
('Batterie', 'number') \
]
def sendAnswer(conn):
#protect against the data supplier
Mutex.acquire()
Trans = deepcopy(DatArr)
Mutex.release()
#create and populate the DataTable
data_table = gviz_api.DataTable(DESCRIPTION)
data_table.LoadData(Trans)
Answer = data_table.ToJSon()
#send ti to the webserver
lng = len(Answer)
try:
conn.sendall(bytes("L{:06d};".format(lng),"UTF-8"))
conn.sendall(bytes(Answer,"UTF-8"))
except BaseException:
# if anything goes wrong, try again next time
pass
def on_message(client, userdata, message):
global Last, DatArr
#get the data from the broker
cur = json.loads(str(message.payload, encoding='utf-8'))
if cur == Last and len(DatArr)>2 : return
now = datetime.now()
# protect against the webserver
Mutex.acquire()
#add the data
DatArr.append([now, cur["temp"], cur["hum"], cur["bat"]])
#cleanup old values
Last = cur
for i in range(len(DatArr)):
if now - DatArr[0][0] > timedelta(days=1):
DatArr.pop(0)
else:
break
Mutex.release()
This works, but instead of keeping the data in the python variable I want to persist it in a file (preferrably JSON). But I cannot JSON.dump() a datetime variable and I cannot .LoadData() a string into a gviz DataTable. The python gviz also lacks an "addRow()". Any suggestions?
Much thanks in advance!
Based on the answers to this question: JSON datetime between Python and JavaScript
I found a solution and implemented it in a python module:
import json
import datetime
class DateTimeJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return dict(nested_datetime=obj.isoformat())
else:
return super(DateTimeJSONEncoder, self).default(obj)
def datetime_decoder(d):
if len(d) == 1 and 'nested_datetime' in d:
return datetime.datetime.strptime(d['nested_datetime'], '%Y-%m-%dT%H:%M:%S.%f')
result = {}
for prop in d:
if isinstance(d[prop], dict):
result[prop] = datetime_decoder(d[prop])
else:
result[prop] = d[prop]
return result
The class and the function go as named parameters into the json.dump and json.load functions like this:
DatArr = json.load(DatFile, object_hook=djson.datetime_decoder)
and
json.dump(DatArr, DatFile, cls=djson.DateTimeJSONEncoder)
This persists the formerly global variable DatArr in the json file DatFile.
Thanks to all the posters to the above question for providing the information.
(A same question is available in Stackoverflow. But that didn't help me because it used other function)
API Documentation
Hello, I am trying to implement Opensubtitle API with Python. I prefer trying to search subtitle file with hash, because it's accurate.
As I have never used xmlrpc before and quite new to using APIs, I had to study to make it work. But I am stuck at the final point. My program is returning Status 200 (OK), but the 'data' array is blank. I think, I am doing something wrong with the paramater passing thing. The code is here:
from xmlrpclib import ServerProxy
import hashCheck, os
server = 'http://api.opensubtitles.org/xml-rpc'
class MainEngine(object):
def __init__(self, language="None"):
self.rpc = ServerProxy(server, allow_none=True)
user_agent = 'OSTestUserAgentTemp'
self.Start()
def getToken(self):
self.logindata = self.rpc.LogIn(None, None, "en", "OSTestUserAgentTemp")
self.Token = self.logindata["token"]
return self.Token
def subSearch(self, path):
self.hash = hashCheck.hash(self.path)
token = self.getToken()
self.param = [
token, # token
[
'eng', # sublanguageid
self.hash, #hash
os.path.getsize(path), # byte size
]
]
Obj = self.rpc.SearchSubtitles(token, self.param)
print Obj
def Start(self):
# print server
self.path = "E:\Movies\English\Captain Phillips\Captain Phillips.mp4"
self.subSearch(self.path)
def main():
MainEngine()
if __name__ == '__main__':
main()
There is no sys.getlinuxversion(), and I get the feeling I may have to piece it together from a few places (which is fine); but, maybe not?
import sys, os
from collections import namedtuple
## This defines an interface similar to the object that getwindowsversion() returns
## (I say 'similar' so I don't have to explain the descriptor magic behind it)
WindowsVersion = namedtuple('WindowsVersion', 'major minor build platform product_type service_pack service_pack_major service_pack_minor suite_mask')
## Looking for something that quacks like this:
LinuxVersion = namedtuple('LinuxVersion', 'major minor build whatever attrs are needed')
## To create something that quacks similarly to this (in case you're wondering)
PlatformVersion = namedtuple('PlatformVersion', 'major minor micro build info')
## EDIT: A newer version of Version, perhaps?
class Version:
_fields = None
def __init__(self, *subversions, **info)
self.version = '.'.join([str(sub) for sub in subversions])
self.info = info
instance_fields = self.__class__._fields
if instance_fields is not None:
if isinstance(instance_fields, str):
instance_fields = instance_fields.split()
for field in instance_fields:
if field in self.info:
setattr(self, field, self.info.pop(field))
## This makes the PlatformVersion (or any other Version) definition a mere:
class PlatformVersion(Version):
_fields = 'build' ## :)
## Now PlatformInfo looks something like:
class PlatformInfo:
def __init__(self, *version_number, platform=None, version_info=None, **info):
self.platform = sys.platform if platform is None else platform
self.info = info
if self.platform in ('win32', 'win64'):
works_great = sys.getwindowsversion()
self.version = PlatformVersion(works_great.major, works_great.minor, works_great.service_pack_major, build=works_great.build, **dict(version_info))
else:
self.version = your_answer(PlatformVersion(os.uname().release, build=os.uname().version, **dict(version_info))), 'hopefully'
self.version = self.version[0] ## lol
Thanks!
Linux (and all Unix-like systems) have uname syscall that provide such information:
>>> import os
>>> os.uname().release
'3.11.10-17-desktop'
It shows version of kernel.
Note that on Python 2 it will return tuple instead of namedtuple:
>>> import os
>>> os.uname()[2]
'3.11.10-17-desktop'