Develop Reference

using ctypes.memset() in python2.7

I am trying to memset a substring in python file using ctypes.memset
Here is the code I am using:
import ctypes
def memz(string):
buffsize = len(string)+1
size = sys.getsizeof(string)
offset = size - buffsize
start = string.find('{') -----------> 142
end = string.find('}') + 1 --------> 167
location = id(string)
ctypes.memset(location + offset + start, 0, end-start)
I see that this does not memset the substring, but writes some other part of memory. I suspect I am not passing the correct memory location to ctypes.memset.
Do I need to change the format of the location(location + offset + start) that I am passing to ctypes.memset?
PS: The solution is derived from Mark data as sensitive in python. I tried that solution but using the memset from ctypes.CDLL('libc.so.6').memset results in a seg fault. I am using Python2.7.11.

Your solution works too. I tried running it locally and it cleared the required substring.
str = 'This is a {sample} string'
print(ctypes.string_at(id(str), sys.getsizeof(str)))
buffSize = len(str)+1
size = sys.getsizeof(str)
offset = size - buffSize
start = str.find('{')
end = str.find('}') + 1
location = id(str)
ctypes.memset(location + offset + start, 0, end-start)
print(ctypes.string_at(id(str), sys.getsizeof(str)))
It produces following output
���A,�5~��dThis is a {sample} string
���A,�5~��dThis is a string

remove the last written line and replace it python

In order to create a progress bar that show the lode by present
The thinking behind it was to erase the line and then rewrite it so it will look like the current precent.
I tried to do this:
import time
---some kind of loop---:
... downloading stuff ...
b = str((prhelp / 576)*100)
print(b + "%", end="\r")
time.sleep(1)
prhelp = prhelp + 1
But it just don't printing any thing
Also I would like to know how to print a number only 3 digits after the dot -
12.345678 --> 12.345
So my program won't print too long precent

it's because when you print, you give the parameter end is the string that will be added to the end. By default, it's \n. You gave \r. What this character means is remove the current line. That's why it doesn't print a thing...
If you want it to work, your replace this:
print(b + "%", end="\r")
by:
print(b + "%")
What I'm guessing you want is:
import time
print('Hello', end='')
time.sleep(1)
print('\rworld!')
This code will print Hello, wait 1 second, and then replace the Hello by world!

At the end I just needed to do:
import time
b = str(round((prhelp / 576) * 100,3))
print('\r',end = "")
print(b + "%",end = "")
time.sleep(0.01)
prhelp = prhelp + 1
By doing
print('\r',end = "")
I'm removing the last line and makeing place for the next number im writing.
Thanks for all the helpers

renaming pcraster mapstack

I have a folder filled with 20 years precipitation pcraster mapstack in days, I've managed to extract from the original netcdf file precipitation value for my interest area and rename it into this to avoid confusion
precip.19810101
precip.19810102
precip.19810103
precip.19810104
precip.19810105
...
precip.20111231
but after that, I want to rename all of my files into pcraster mapstack based on this sequence of dates
precip00.001
precip00.002
precip00.003
precip00.004
...
I'm a beginner in python, is there any help or example for me to figure it out how to do this?
Thank you

Here's something I put together, based on some old Python scripts I once wrote:
#! /usr/bin/env python
# Rename PCRaster map stack with names following prefix.yyymmmdd to stack with valid
# PCRaster time step numbers
# Johan van der Knijff
#
# Example input stack:
#
# precip.19810101
# precip.19810102
# precip.19810103
# precip.19810104
# precip.19810105
#
# Then run script with following arguments:
#
# python renpcrstack.py precip 1
#
# Result:
#
# precip00.001
# precip00.002
# precip00.003
# precip00.004
# precip00.005
#
import sys
import os
import argparse
import math
import datetime
import glob
# Create argument parser
parser = argparse.ArgumentParser(
description="Rename map stack")
def parseCommandLine():
# Add arguments
parser.add_argument('prefix',
action="store",
type=str,
help="prefix of input map stack (also used as output prefix)")
parser.add_argument('stepStartOut',
action="store",
type=int,
help="time step number that is assigned to first map in output stack")
# Parse arguments
args = parser.parse_args()
return(args)
def dateToJulianDay(date):
# Calculate Julian Day from date
# Source: https://en.wikipedia.org/wiki/Julian_day#Converting_Julian_or_Gregorian_calendar_date_to_Julian_day_number
a = (14 - date.month)/12
y = date.year + 4800 - a
m = date.month +12*a - 3
JulianDay = date.day + math.floor((153*m + 2)/5) + 365*y + math.floor(y/4) \
- math.floor(y/100) + math.floor(y/400) - 32045
return(JulianDay)
def genStackNames(prefix,start,end, stepSize):
# Generate list with names of all maps
# map name is made up of 11 characters, and chars 8 and 9 are
# separated by a dot. Name starts with prefix, ends with time step
# number and all character positions in between are filled with zeroes
# define list that will contain map names
listMaps = []
# Count no chars prefix
charsPrefix = len(prefix)
# Maximum no chars needed for suffix (end step)
maxCharsSuffix = len(str(end))
# No of free positions between pre- and suffix
noFreePositions = 11 - charsPrefix - maxCharsSuffix
# Trim prefix if not enough character positions are available
if noFreePositions < 0:
# No of chars to cut from prefix if 11-char limit is exceeded
charsToCut = charsPrefix + maxCharsSuffix - 11
charsToKeep = charsPrefix - charsToCut
# Updated prefix
prefix = prefix[0:charsToKeep]
# Updated prefix length
charsPrefix = len(prefix)
# Generate name for each step
for i in range(start,end + 1,stepSize):
# No of chars in suffix for this step
charsSuffix = len(str(i))
# No of zeroes to fill
noZeroes = 11 - charsPrefix - charsSuffix
# Total no of chars right of prefix
charsAfterPrefix = noZeroes + charsSuffix
# Name of map
thisName = prefix + (str(i)).zfill(charsAfterPrefix)
thisFile = thisName[0:8]+"." + thisName[8:11]
listMaps.append(thisFile)
return listMaps
def main():
# Parse command line arguments
args = parseCommandLine()
prefix = args.prefix
stepStartOut = args.stepStartOut
# Glob pattern for input maps: prefix + dot + 8 char extension
pattern = prefix + ".????????"
# Get list of all input maps based on glob pattern
mapsIn = glob.glob(pattern)
# Set time format
tfmt = "%Y%m%d"
# Set up dictionary that will act as lookup table between Julian Days (key)
# and Date string
jDayDate = {}
for map in mapsIn:
baseNameIn = os.path.splitext(map)[0]
dateIn = os.path.splitext(map)[1].strip(".")
# Convert to date / time format
dt = datetime.datetime.strptime(dateIn, tfmt)
# Convert date to Julian day number
jDay = int(dateToJulianDay(dt))
# Store as key-value pair in dictionary
jDayDate[jDay] = dateIn
# Number of input maps (equals number of key-value pairs)
noMaps = len(jDayDate)
# Create list of names for output files
mapNamesOut = genStackNames(prefix, stepStartOut, noMaps + stepStartOut -1, 1)
# Iterate over Julian Days (ascending order)
i = 0
for key in sorted(jDayDate):
# Name of input file
fileIn = prefix + "."+ jDayDate[key]
# Name of output file
fileOut = mapNamesOut[i]
# Rename file
os.rename(fileIn, fileOut)
print("Renamed " + fileIn + " ---> " + fileOut)
i += 1
main()
(Alternatively download the code from my Github Gist.)
You can run it from the command line, using the prefix of your map stack and the number of the first output map as arguments, e.g.:
python renpcrmaps.py precip 1
Please note that the script renames the files in place, so make sure to make a copy of your original map stack in case something goes wrong (I only did some very limited testing on this!).
Also, the script assumes a non-sparse input map stack, i.e. in case of daily maps, an input map exists for each day. In case of missing days, the numbering of the output maps will not be what you'd expect.
The internal conversion of all dates to Julian Days may be a bit overkill here, but once you start doing more advanced transformations it does make things easier because it gives you decimal numbers which are more straightforward to manipulate than date strings.

as you gave the [batch-file] tag, I assume, Batch is ok:
#echo off
setlocal enabledelayedexpansion
set /a counti=0
for /f "delims=" %%a in ('dir /b /on precip.*') do (
set /a counti+=1
set "counts=000000000!counti!"
ECHO ren "%%a" "precip!counts:~-6,3!.!counts:~-3!"
)
remove the ECHO after successfully checking the Output
EDITED to match your precip00.999 is precip01.000 ... until precip07.300 requirement (in your question it's precip000.001 in your comment it's precip00.001 - I decided to use the first Format, can easily be changed to ECHO ren "%%a" "precip!counts:~-5,2!.!counts:~-3!" for the second Format.). Although it's not Batch anymore, I'll leave the answer, maybe you can at least use the logic.
If you are not firm with Batch, the %variable:~-6,3% Syntax is explained with set /?

I've faced this issue a short while ago. Please note I am new both to python and PCRaster so do not take me example without check.
import os
import shutil
import fnmatch
import subprocess
from os import listdir
from os.path import isfile, join
from shutil import copyfile
TipeofFile = 'precip.????????' # original file
Files = []
for iListFile in sorted(os.listdir('.')):
if fnmatch.fnmatch(iListFile, TipeofFile):
Files.append(iListFile)
digiafter = 3 #after the point: .001, .002, 0.003
digitTotal = 8 #total: precipi00000.000 (5.3)
for j in xrange(0, len(Files)):
num = str(j + 1)
nameFile = Files[j]
putZeros = digitTotal - len(num)
for x in xrange(0,putZeros):
num = "0" + num
precip = num[0:digitTotal-digiafter]+ '.' +num[digitTotal-digiafter:digitTotal]
precip = str(precip)
precip = 'precip' + precip
copyfile(nameFile, precip)

Python - Trying to check input against range, then convert the variable into a string to be concatenated

first time posting here. I have searched high and low to figure out a way to do this, but I either don't understand how to apply existing answers to my code.
What I am trying to do is this: I want to take user input (a year), make sure it's between a range of years, and then if it is, concatenate it with an existing string as a variable.
The result of this code is that I get through and give all the necessary inputs, and then it fails with "fullInstr = str("cp -r /mnt/data/archive"+ fquery+ "/" + yquery+mquery+dquery+"/"+hquery+"*" + " " + outl2)
TypeError: cannot concatenate 'str' and 'int' objects"
import os
import sys
os.system("clear")
overW = str("0")
outf = str("/autocopy.sh") # Output file full path and file name
if os.path.isfile("/autocopy.sh"): #
overW = raw_input("This will overwrite the previous version. Do you want to continue? (y/n) ")
os.system("clear")
else:
os.system("clear")
if overW != "y":
os.system("clear")
sys.exit("No changes made.\n\n")
else:
os.system("clear")
#! Could also prompt for output file name, depending on how army-proof this needs to be.
finishMessage = "Finished."
outl = str("0") # Copy-to location
outl2 = str("0") # Modified Copy-to location
fquery = str("0") # A or B location variable
yquery = int("0") # Year variable
mquery = int("0") # Month variable
dquery = int("0") # Day variable
hquery = str("0") # Hour variable
mh1 = int("0") # Modified starting hour after transformation
mh2 = int("0") # Modified ending hour after transformation
mpath = str("0") # makes path if needed
fullInstr = str("0") # Full command set to write to file
formatList = (['A', 'B'])
yearList = list(range(2000,2099))
#monthList = (['01']-['12'])
#! hquery is going to have to parse for ranges
# Instruction header
print "Builds a script to automatically copy folders and files from the storage array to a location of your choosing.\n"
print "Valid inputs for the questions below are numeric."
print "Year would be the full year, i.e. 2013"
print "Month is the two-digit month, i.e. 10"
print "Day is the two-digit day, i.e. 22"
print "Hour or hour range is just the first two digits of the hours you want to copy. i.e. 15 or 00-23\n\n"
outl = raw_input("Where do you want to copy the files to? Type the full path: ")
while not os.path.exists(outl):
mpath = raw_input ("\nThat path doesn't exist on this system. Do you want to create it? (y/n) ")
if mpath != "y":
outl = raw_input("Where do you want to copy the files to? Type a valid path: ")
else:
os.mkdir(outl)
print "\n"
if not outl.endswith("/"):
outl2 = outl + "/"
fquery = raw_input("Do you want to copy A or B? ")
while not(fquery in formatList):
print "\nInvalid input. You have to choose one of the two as printed above."
fquery = raw_input("\nDo you want to copy A or B? ")
print "\n"
yquery = int(raw_input("What year? "))
while yquery not in yearList:
print "\nInvalid input. You have to choose a year in this century."
yquery = int(raw_input("\nWhat year? "))
print "\n"
mquery = raw_input("What day? ")
#! Valid months are 01 to 12
dquery = raw_input("What day? ")
#! Valid days are 01 to 31
hquery = raw_input("What hour or hour range? ")
#! if input is greater than two characters is must contain a - character
#! if is not 00-23, separate first and last numbers and write a line for each. If it isn't a range, don't transform it.
#os.system("touch " + outf)
#! if no error keep going
fullInstr = str("cp -r /mnt/data/archive"+ fquery+ "/" + yquery+mquery+dquery+"/"+hquery+"*" + " " + outl2)
os.system("echo "+fullInstr+ "> /autocopy.sh")
#os.system("chmod u+x "+outf) # Makes the output file executable
#! if error = 0 set finishMessage to print "Your shell script is complete and is ready to run. Run it by typing ." + outf
#! if error is <> 0 set finishMessage to print "Wasn't able to make the output file executable automatically. Use chmod to modify the file permissions manually on the file "+outf
#os.system("clear")
print finishMessage+"\n\n"
Stuff that is commented out is either not working or not implemented yet. I know the code quality is probably not the best, but this is my first time coding to do something I require. I have tried a lot of things like yquery = str(yquery) or just changing the fullInstr string to have str(yquery) in it and I can not get it to work. I am becoming frustrated.

You should use format for string formatting to avoid worrying about the type of variable being concatenated.
fullInstr = "cp -r /mnt/data/archive/{0}/{1}{2}{3}/{4}* {5}".format(fquery,yquery,mquery,dquery,hquery,out12)

That is because you need to convert the int to a str object.
fullInstr = str("cp -r /mnt/data/archive" + fquery + "/" + str(yquery) + mquery + dquery + "/" + hquery + "*" + " " + ^
outl2) ^
# ^ -> Use str to convert
The above code worked on my machine. Running windows 8. Python 2.7.5. I just had to change clear to cls.

Python: Simple script that parses metrics data

I have a small Python script that I need to modify because the format of the metrics file has changed slightly. I do not know Python at all and have tried to take an honest effort to fix it myself. The changes make sense to me but apparently there is still one issue with the script. Otherwise, everything else is working. Here's what the script looks like:
import sys
import datetime
##########################################################################
now = datetime.datetime.now();
logFile = now.strftime("%Y%m%d")+'.QE-Metric.log';
underlyingParse = True;
strParse = "UNDERLYING_TICK";
if (len(sys.argv) == 2):
if sys.argv[1] == '2':
strParse = "ORDER_SHOOT";
underlyingParse = False;
elif (len(sys.argv) == 3):
logFile = sys.argv[2];
if sys.argv[1] == '2':
strParse = "ORDER_SHOOT";
underlyingParse = False;
else:
print 'Incorrect number of arguments. Usage: <exec> <mode (1) Underlying (2) OrderShoot> <FileName (optional)>'
sys.exit()
##########################################################################
# Read the deployment file
FIput = open(logFile, 'r');
FOput = open('ParsedMetrics.txt', 'w');
##########################################################################
def ParseMetrics( file_lines ):
ii = 0
tokens = [];
for ii in range(len(file_lines)):
line = file_lines[ii].strip()
if (line.find(strParse) != -1):
tokens = line.split(",");
currentTime = float(tokens[2])
if (underlyingParse == True and ii != 0):
newIndex = ii-1
prevLine = file_lines[newIndex].strip()
while (prevLine.find("ORDER_SHOOT") != -1 and newIndex > -1):
newIndex -= 1;
tokens = prevLine.split(",");
currentTime -= float(tokens[2]);
prevLine = file_lines[newIndex].strip();
if currentTime > 0:
FOput.write(str(currentTime) + '\n')
##########################################################################
file_lines = FIput.readlines()
ParseMetrics( file_lines );
print 'Metrics parsed and written to ParsedMetrics.txt'
Everything is working fine except for the logic that is supposed to reverse iterate through previous lines to add up the ORDER_SHOOT numbers since the last UNDERLYING_TICK event occurred (starting at the code: if (underlyingParse == True and ii != 0):...) and then subtract that total from the current UNDERLYING_TICK event line being processed. This is what a typical line in the file being parsed looks like:
08:40:02.039387(+26): UNDERLYING_TICK, 1377, 1499.89
Basically, I'm only interested in the last data element (1499.89) which is the time in micros. I know it has to be something stupid. I just need another pair of eyes. Thanks!

So, if command line option is 2, the function creates an output file where all the lines contain just the 'time' portion of the lines from the input file that had the "order_shoot" token in them?
And if the command line option is 1, the function creates an output file with a line for each line in input file that contained the 'underlying_tick' token, except that the number you want here is the underlying_tick time value minus all the order_shoot time values that occurred SINCE the preceding underlying_tick value (or from the start of file if this is the first one)?
If this is correct, and all lines are unique (there are no duplicates), then I would suggest the following re-written script:
#### Imports unchanged.
import sys
import datetime
#### Changing the error checking to be a little simpler.
#### If the number of args is wrong, or the "mode" arg is
#### not a valid option, it will print the error message
#### and exit.
if len(sys.argv) not in (2,3) or sys.argv[2] not in (1,2):
print 'Incorrect arguments. Usage: <exec> <mode (1) Underlying (2) OrderShoot> <FileName (optional)>'
sys.exit()
#### the default previously specified in the original code.
now = datetime.datetime.now()
#### Using ternary logic to set the input file to either
#### the files specified in argv[2] (if it exists), or to
#### the default previously specified in the original code.
FIput = open((sys.argv[2] if len(sys.argv)==3
else now.strftime("%Y%m%d")+'.QE-Metric.log'), 'r');
#### Output file not changed.
FOput = open('ParsedMetrics.txt', 'w');
#### START RE-WRITTEN FUNCTION
def ParseMetrics(file_lines,mode):
#### The function now takes two params - the lines from the
#### input file, and the 'mode' - whichever the user selected
#### at run-time. As you can see from the call down below, this
#### is taken straight from argv[1].
if mode == '1':
#### So if we're doing underlying_tick mode, we want to find each tick,
#### then for each tick, sum the preceding order_shoots since the last
#### tick (or start of file for the first tick).
ticks = [file_lines.index(line) for line in file_lines \
if 'UNDERLYING_TICK' in line]
#### The above list comprehension iterates over file_lines, and creates
#### a list of the indexes to file_lines elements that contain ticks.
####
#### Then the following loop iterates over ticks, and for each tick,
#### subtracts the sum of all times for order_shoots that occure prior
#### to the tick, from the time value of the tick itself. Then that
#### value is written to the outfile.
for tick in ticks:
sub_time = float(file_lines[tick].split(",")[2]) - \
sum([float(line.split(",")[2]) \
for line in file_lines if "ORDER_SHOOT" in line \
and file_lines.index(line) <= tick]
FOput.write(float(line.split(",")[2]))
#### if the mode is 2, then it just runs through file_lines and
#### outputs all of the order_shoot time values.
if mode == '2':
for line in file_lines:
if 'ORDER_SHOOT' in line:
FOput.write(float(line.split(",")[2]))
#### END OF REWRITTEN FUNCTION
#### As you can see immediately below, we pass sys.argv[2] for the
#### mode argument of the ParseMetrics function.
ParseMetrics(FIput.readlines(),sys.argv[2])
print 'Metrics parsed and written to ParsedMetrics.txt'
And that should do the trick. The main issue is that if you have any lines with "UNDERLYING_TICK" that are exact duplicates of any other such line, then this will not work. Different logic would need to be applied to get the correct indexes.
I am sure there is a way to make this much better, but this was my first thought.
It's also worth noting I added a lot of inline line breaks to the above source for readability, but you might want to pull them if you use this as written.

It's unclear what is wrong with your output because you don't show your output and we can't really understand your input.
I am assuming the following:
Lines are formatted as "absolutetime: TYPE, positiveinteger, float_time_duration_in_ms", where this last item is the amount of time the thing took.
Lines are sorted by "absolutetime". As a consequence, the ORDER_SHOOTs that belong to an UNDERLYING_TICK are always on the lines since the last UNDERLYING_TICK (or the beginning of the file), and only those lines. If this assumption is not true, then you need to sort the file first. You can either do that with a separate program (e.g. pipe output from sort), or use the bisect module to store your lines sorted and easily extract the relevant lines.
If both these assumptions are true, take a look at the following script instead. (Untested because I don't have a big input sample or an output sample to compare against.)
This is a much more Pythonic style, much easier to read and understand, doesn't make use of global variables as function parameters, and should be much more efficient because it doesn't iterate backwards through lines or load the entire file into memory to parse it.
It also demonstrates use of the argparse module for your command line parsing. This isn't necessary, but if you have a lot of command-line Python scripts you should get familiar with it.
import sys
VALIDTYPES = ['UNDERLYING_TICK','ORDER_SHOOT']
def parseLine(line):
# format of `tokens`:
# 0 = absolute timestamp
# 1 = event type
# 2 = ???
# 3 = timedelta (microseconds)
tokens = [t.strip(':, \t') for t in line.strip().split()]
if tokens[1] not in VALIDTYPES:
return None
tokens[2] = int(tokens[2])
tokens[3] = float(tokens[3])
return tuple(tokens)
def parseMetrics(lines, parsetype):
"""Yield timedelta for each line of specified type
If parsetype is 'UNDERLYING_TICK', subtract previous ORDER_SHOOT
timedeltas from the current UNDERLYING_TICK delta before yielding
"""
order_shoots_between_ticks = []
for line in lines:
tokens = parseLine(line)
if tokens is None:
continue # go home early
if parsetype=='UNDERLYING_TICK':
if tokens[1]=='ORDER_SHOOT':
order_shoots_between_ticks.append(tokens)
elif tokens[1]=='UNDERLYING_TICK':
adjustedtick = tokens[3] - sum(t[3] for t in order_shoots_between_ticks)
order_shoots_between_ticks = []
yield adjustedtick
elif parsetype==tokens[1]:
yield tokens[3]
def parseFile(instream, outstream, parsetype):
printablelines = ("{0:f}\n".format(time) for time in parseMetrics(instream, parsetype))
outstream.writelines(printablelines)
def main(argv):
import argparse, datetime
parser = argparse.ArgumentParser(description='Output timedeltas from a QE-Metric log file')
parser.add_argument('mode', type=int, choices=range(1, len(VALIDTYPES)+1),
help="the types to parse. Valid values are: 1 (Underlying), 2 (OrderShoot)")
parser.add_argument('infile', required=False,
default='{}.QE-Metric.log'.format(datetime.datetime.now().strftime('%Y%m%d'))
help="the input file. Defaults to today's file: YYYYMMDD.QE-Metric.log. Use - for stdin.")
parser.add_argument('outfile', required=False,
default='ParsedMetrics.txt',
help="the output file. Defaults to ParsedMetrics.txt. Use - for stdout.")
parser.add_argument('--verbose', '-v', action='store_true')
args = parser.parse_args(argv)
args.mode = VALIDTYPES[args.mode-1]
if args.infile=='-':
instream = sys.stdin
else:
instream = open(args.infile, 'rb')
if args.outfile=='-':
outstream = sys.stdout
else:
outstream = open(args.outfile, 'wb')
parseFile(instream, outstream, args.mode)
instream.close()
outstream.close()
if args.verbose:
sys.stderr.write('Metrics parsed and written to {0}\n'.format(args.outfile))
if __name__=='__main__':
main(sys.argv[1:])