read a multi tier csv file in python - python

I need to read the following data out of a text file;
[L02]
g,g,g,g,g,g,g,g,g,g,w,w,w,w,g,g
g,g,g,g,g,g,g,g,g,w,w,w,w,w,g,g
g,g,g,g,g,g,g,g,w,w,w,w,w,g,g,g
g,g,g,g,g,g,g,g,w,w,w,w,g,g,g,g
g,g,g,g,g,g,g,g,g,w,w,w,w,g,g,g
g,g,g,g,g,g,g,g,g,g,w,w,w,w,g,g
g,g,g,g,g,g,g,g,g,g,g,w,w,w,g,g
g,g,g,g,g,g,g,g,g,g,g,w,w,g,g,g
g,g,g,g,g,g,g,g,g,g,g,w,w,g,g,g
g,g,g,g,g,g,g,g,g,g,w,w,w,g,g,g
g,g,g,g,g,g,g,g,g,w,w,w,g,g,g,g
g,g,g,g,g,g,g,g,w,w,w,w,g,g,g,g
g,g,g,g,g,g,g,w,w,w,w,g,g,g,g,g
g,g,g,g,g,g,g,w,w,w,g,g,g,g,g,g
g,g,g,g,g,g,w,w,w,w,w,g,g,g,g,g
g,g,g,g,g,g,g,w,w,w,w,g,g,g,g,g
[L01]
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
d,d,d,d,d,d,d,d,d,d,d,d,d,d,d,d
I can read a single block as a csv file but I don't know how to read each file as a separate list
The output I want is to have arrays/lists for each block with the block contents as the list elements. Any ideas?

Here's a script that demonstrates how to break down the problem into reusable steps (functions) and performs the transformation your need.
import itertools
import operator
import re
import csv
import pprint
class TaggedLine(str):
"""
Override str to allow a tag to be added.
"""
def __new__(cls, val, tag):
return str.__new__(cls, val)
def __init__(self, val, tag):
super(TaggedLine, self).__init__(val)
self.tag = tag
def sections(stream):
"""
Tag each line of the stream with its [section] (or None)
"""
section_pattern = re.compile('\[(.*)\]')
section = None
for line in stream:
matcher = section_pattern.match(line)
if matcher:
section = matcher.group(1)
continue
yield TaggedLine(line, section)
def splitter(stream):
"""
Group each stream into sections
"""
return itertools.groupby(sections(stream), operator.attrgetter('tag'))
def parsed_sections(stream):
for section, lines in splitter(stream):
yield section, list(csv.reader(lines))
if __name__ == '__main__':
with open('data.csv') as stream:
for section, data in parsed_sections(stream):
print 'section', section
pprint.pprint(data[:2])
Save your file as 'data.csv' and the script will run on your data with this output:
section L02
[['g',
'g',
'g',
'g',
'g',
'g',
'g',
'g',
'g',
'g',
'w',
'w',
'w',
'w',
'g',
'g'],
['g',
'g',
'g',
'g',
'g',
'g',
'g',
'g',
'g',
'w',
'w',
'w',
'w',
'w',
'g',
'g']]
section L01
[['d',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd'],
['d',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd',
'd']]

If you have numpy, you could read the file into a numpy array. comments='[' tells np.genfromtxt to ignore lines that begin with [. The reshape method places each 16x16 block in its own "layer".
import numpy as np
arr=np.genfromtxt('data.csv',comments='[',delimiter=',',dtype=None)
arr=arr.reshape(-1,16,16)
You can access the nth layer with arr[n].

Related

There is a problem with the writing of regular expressions. If there is a problem with the following writing, how can I modify it

self.parameters[['设备机型', 'DeviceType', 'T6-800'], ['顶升抱闸', 'LiftBreak', '1'],
['电机方向', 'MotorDir', '1']]
self.text("[10-20 10:15:44][2584][para_config_cmd][000] DeviceType:800
[10-20 10:15:44][2584][para_config_cmd][001] RobotID:613
[10-20 10:15:44][2584][para_config_cmd][002] LiftBreak:1
[10-20 10:15:45][2584][para_config_cmd][003] WiFiEnable:1
[10-20 10:15:45][2584][para_config_cmd][006] SelfCaliSpeed:300
[10-20 10:15:45][2584][para_config_cmd][007] PTTimeSecond:60
")
paralog = []
for i in range(len(self.parameters)):
if self.parameters[i][1] in self.text:
self.c = re.search(''+ self.parameters[i][1]+'(:.+?)[\],\n]' , self.text)
self.paralog.append(self.c.group())
for k in range(len(self.paralog)):
self.paralog[k] = list(self.paralog[k])
Running results:
[['D', 'e', 'v', 'i', 'c', 'e', 'T', 'y', 'p', 'e', ':', '8', '0', '0', '\n'], ['L', 'i', 'f', 't', 'B', 'r', 'e', 'a', 'k', ':', '1', '\n']]
Want results:
[[DeviceType:800],[DeviceType:800]]
self references removed, but this list comprehension returns what you're looking for.
import re
parameters = [['设备机型', 'DeviceType', 'T6-800'], ['顶升抱闸', 'LiftBreak', '1'], ['电机方向', 'MotorDir', '1']]
text = ("""
[10-20 10:15:44][2584][para_config_cmd][000] DeviceType:800
[10-20 10:15:44][2584][para_config_cmd][001] RobotID:613
[10-20 10:15:44][2584][para_config_cmd][002] LiftBreak:1
[10-20 10:15:45][2584][para_config_cmd][003] WiFiEnable:1
[10-20 10:15:45][2584][para_config_cmd][006] SelfCaliSpeed:300
[10-20 10:15:45][2584][para_config_cmd][007] PTTimeSecond:60
""")
paralog = [parameter for parameter in re.findall(r'\b([A-Za-z]\w+:\d+)',text)
if any(parameters_check[1] in parameter
for parameters_check in parameters) ]
print(paralog)
Output
['DeviceType:800', 'LiftBreak:1']

When I Run My Code I Get A Huge Error In Python

If I Run This Code:
import re
mapping = {'gde1': 'a', '4&TW': 'b', 'E#ZB': 'c', 'B0F&': 'd', 'uvRD': 'e', 'M7vr': 'f', 'X$*d': 'g', 'XQ%R': 'h', 'jma+': 'i', 'P9We': 'j', 'xw4O': 'k', 'zT#3': 'l', '4B30': 'm', 'Jc&$': 'n', 'Szb7': 'o', '+yvf': 'p', 'tGgQ': 'q', '$g0Y': 'r', '9y6h': 's', '%Mo&': 't', '*fyQ': 'u', 'jaDZ': 'v', '%FaT': 'w', 'FBn5': 'x', 'sX2s': 'y', 'bFwN': 'z', 'qV0c': 'A', 'Zwg0': 'B', 'v!6!': 'C', 'wKKw': 'D', 'v3xd': 'E', 'oy#u': 'F', 'bu+S': 'G', '4Sh2': 'H', '6drQ': 'I', 'oQ#w': 'J', '555k': 'K', 'G$d3': 'L', 'AuJ3': 'M', 'XGm+': 'N', 'aqs3': 'O', '9Tvb': 'P', 'H%42': 'Q', 'g8E+': 'R', 'kwgO': 'S', '86+y': 'T', 'bv5e': 'U', 'Hg4b': 'V', '=dsc': 'W', 'B6jk': 'X', '#Av9': 'Y', 'kwPS': 'Z', 'Y1q$': '1', 'bNny': '2', 'mu!R': '3', 'HHRK': '4', 'm$!V': '5', '71D$': '6', 'm6qA': '7', 'o#U4': '8', '#ze1': '9', 'aAM6': '0', 'u7$q': '~', 'jwzQ': ':', 'yw#S': "'", '65BO': '+', 'xAhB': '[', 'X=G+': '\\', 'EXa7': '#', 'WpX6': '^', 'C#c2': '{', 'C8!y': '%', 'rCEB': '(', 'Ebos': '-', 'VKb!': '"', 'WPOu': '*', '9Ht$': '|', 'ZHo=': ',', 'ZkKY': '&', 'artq': '<', 'nS9m': '`', 'k!vX': '}', 'j1f6': '.', 'VpV%': '_', '5u*$': '=', '8rjs': ']', 'JJc!': '!', 'SW%$': '>', '#g#G': ';', '#xF&': '?', '$E60': '#', 'U7Su': '$', 'g8FA': ')', '4o4s': '/'}
string = 'gde1'
choices = f'({"|".join(mapping)})'
result = ''.join(mapping.get(s, s) for s in re.split(choices, string))
print(result)
This Code Is Supposed To Output 'a'.
But It Gives Me This Error Message:
Traceback (most recent call last):
File "c:\EDS\fatstrst.py", line 5, in <module>
result = ''.join(mapping.get(s, s) for s in re.split(choices, string))
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\re.py", line 231, in split
return _compile(pattern, flags).split(string, maxsplit)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\re.py", line 304, in _compile
p = sre_compile.compile(pattern, flags)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_compile.py", line 764, in compile
p = sre_parse.parse(p, flags)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_parse.py", line 950, in parse
p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_parse.py", line 443, in _parse_sub
itemsappend(_parse(source, state, verbose, nested + 1,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_parse.py", line 836, in _parse
p = _parse_sub(source, state, sub_verbose, nested + 1)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_parse.py", line 443, in _parse_sub
itemsappend(_parse(source, state, verbose, nested + 1,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.9_3.9.3312.0_x64__qbz5n2kfra8p0\lib\sre_parse.py", line 668, in _parse
raise source.error("nothing to repeat",
re.error: nothing to repeat at position 33
But When I Shorten The mapping = To:
mapping = {'gde1': 'a', '4&TW': 'b', 'E#ZB': 'c', 'B0F&': 'd'}
It Works Fine And The Output Is a.
Does Anyone Have Any Idea On How To Make This Work? Any Help Would Be Greatly Appreciated.
In your mapping you have special regex characters, such as + that indicates that the previous character/group can be repeated.
Your error is due to strings like +yvf where there is nothing before the +. Even if there was no error, you would match the wrong things (e.g. Jc&$ would match Jc& only in the end of the string)
You need to escape those characters. An easy way is to use re.escape:
string = 'gde1'
choices = f'({"|".join(map(re.escape, mapping))})'
result = ''.join(mapping.get(s, s) for s in re.split(choices, string))
print(result)
A quick solution to your issue is to add a backslash \ in front of the regex characters (e.g., * and +).
The modified code is:
import re
mapping = {
'gde1': 'a', '4&TW': 'b', 'E#ZB': 'c', 'B0F&': 'd', 'uvRD': 'e', 'M7vr': 'f', 'X$\*d': 'g', 'XQ%R': 'h', 'jma\+': 'i',
'P9We': 'j', 'xw4O': 'k', 'zT#3': 'l', '4B30': 'm', 'Jc&$': 'n', 'Szb7': 'o', '\+yvf': 'p', 'tGgQ': 'q', '$g0Y': 'r',
'9y6h': 's', '%Mo&': 't', '\*fyQ': 'u', 'jaDZ': 'v', '%FaT': 'w', 'FBn5': 'x', 'sX2s': 'y', 'bFwN': 'z', 'qV0c': 'A',
'Zwg0': 'B', 'v!6!': 'C', 'wKKw': 'D', 'v3xd': 'E', 'oy#u': 'F', 'bu\+S': 'G', '4Sh2': 'H', '6drQ': 'I', 'oQ#w': 'J',
'555k': 'K', 'G$d3': 'L', 'AuJ3': 'M', 'XGm\+': 'N', 'aqs3': 'O', '9Tvb': 'P', 'H%42': 'Q', 'g8E\+': 'R', 'kwgO': 'S',
'86\+y': 'T', 'bv5e': 'U', 'Hg4b': 'V', '=dsc': 'W', 'B6jk': 'X', '#Av9': 'Y', 'kwPS': 'Z', 'Y1q$': '1', 'bNny': '2',
'mu!R': '3', 'HHRK': '4', 'm$!V': '5', '71D$': '6', 'm6qA': '7', 'o#U4': '8', '#ze1': '9', 'aAM6': '0', 'u7$q': '~',
'jwzQ': ':', 'yw#S': "'", '65BO': '\+', 'xAhB': '[', 'X=G\+': '\\', 'EXa7': '#', 'WpX6': '^', 'C#c2': '{', 'C8!y': '%',
'rCEB': '(', 'Ebos': '-', 'VKb!': '"', 'WPOu': '\*', '9Ht$': '|', 'ZHo=': ',', 'ZkKY': '&', 'artq': '<', 'nS9m': '`',
'k!vX': '}', 'j1f6': '.', 'VpV%': '_', '5u\\$': '=', '8rjs': ']', 'JJc!': '!', 'SW%$': '>', '#g#G': ';', '#xF&': '?',
'$E60': '#', 'U7Su': '$', 'g8FA': ')', '4o4s': '/'
}
string = 'gde1'
choices = f'({"|".join(mapping)})'
splits = re.split(choices, string)
result = ''.join(mapping.get(s, s) for s in splits)
print(result)

I'm coding in Python and keep receiving: UnicodeEncodeError: 'ascii' codec can't encode character '\u2019' in position 31: ordinal not in range(128)

The code below is the timeclienthandler.py code that I created in Visual Studio Code. It works sometimes but it still gives me the error if I continuously run the code. I don't understand how it works off and on.
from time import ctime
from threading import Thread
import random
class TimeClientHandler(Thread):
def __init__(self, client):
Thread.__init__(self)
self.client = client
def run(self):
msgList = ["There are some idiots who always answer 'No' to every question, now tell me. Are you one of them?","There's nothing to fear. Except maybe that weird guy over there.","If I'm driving you crazy just remember to put on your seat belt.","I wondered why the baseball was getting bigger. Then it hit me.","You're Just Jealous Because The Voices Are Talking To Me.","Quickest way to get on your feet...miss a car payment.","Why do psychics ask your name?","I'm not opinionated. I'm just always right.","Sanity is the playground for the unimaginative.","It isn't homework unless it's due tomorrow."]
msg = msgList[random.randint(0,len(msgList))]
msge ="\n"+msg
self.client.send(bytes(ctime() + msge,"ascii"))
self.client.close()
The error you specified is occuring at line 18 in your code
Specifically - bytes(ctime() + msge,"ascii")
This error occured because of encoding you specified "ascii" which only supports the following characters:
['\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b', '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '#', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '\x7f']
but the input you are giving to it is '\u2019' or {’}, which is not in the above list hence resulting in the UnicodeEncodeError ('ascii' codec can't encode character '\u2019' in position 0: ordinal not in range(128))
So the solution to this problem is to change the specified encoding (ascii) to (UTF-8)
Try the code below:
from time import ctime
from threading import Thread
import random
class TimeClientHandler(Thread):
def __init__(self, client):
Thread.__init__(self)
self.client = client
def run(self):
msgList = ["There are some idiots who always answer 'No' to every question, now tell me. Are you one of them?","There's nothing to fear. Except maybe that weird guy over there.","If I'm driving you crazy just remember to put on your seat belt.","I wondered why the baseball was getting bigger. Then it hit me.","You're Just Jealous Because The Voices Are Talking To Me.","Quickest way to get on your feet...miss a car payment.","Why do psychics ask your name?","I'm not opinionated. I'm just always right.","Sanity is the playground for the unimaginative.","It isn't homework unless it's due tomorrow."]
msg = msgList[random.randint(0,len(msgList))]
msge ="\n"+msg
self.client.send(bytes(ctime() + msge,"UTF-8"))
self.client.close()
Encoding with UTF-8.
Turn to the folder Lib\site-packages in current used python interpreter, create a file called sitecustomize.py and add the following code in it:
#coding=utf8
import sys
reload(sys)
sys.setdefaultencoding('utf8')
Then restart the VS Code to see if the question goes away.

python preg_replace translate message to gsm formatted message

i am trying to to make a replace in order to send a gsm message later. i use the function below where it takes the message as parameter and convert it to a gsm formatted message.
import re
#staticmethod
def gsm_message(message):
expressions = {
'/[άΆαΑ]/u': 'A',
'/[βΒ]/u': 'B',
'/[έΈεΕ]/u': 'E',
......... more
}
translated_message = re.sub(expressions.keys(), expressions.values(), message)
print(translated_message)
the error i get when i am trying to print is: unhashable type: 'list'.
what can i do to make it work?
The following works in Python3.
import re
def gsm_message(message):
expressions = {
'ά': 'A',
'Ά': 'A',
'α': 'A',
'Α': 'A',
'β': 'B',
'Β': 'B',
'έ': 'E',
'Έ': 'E',
'ε': 'E',
'Ε': 'E',
}
pattern = re.compile('|'.join(expressions.keys()))
translated_message = pattern.sub(lambda x: expressions[x.group()], message)
print(translated_message)
message = "ββααέ"
gsm_message(message)
Gives:
BBAAE

redis: hmset expire, but hgetall can get subkey info

python code:
value = {'sct': 's', 'scn': 'A', 'fct': 'd', 'fcn': 'K', 'bet_value':10}
res = rc.hmset(key, value)
rc.expire(key, 3600)
3600 seconds later
rc.hgetall(key)
result:
{'sct': 's', 'scn': 'A', 'fct': 'd', 'fcn': 'K'}
rc.ttl(key) has no result
The question is that the "key" should be expired after 3600s, but I can still get info with the command "hetall".
The ttl command return None, according to the document it should be -2 or -1
What's wrong? anybody can help?

Categories

Resources