Develop Reference

How to Account and Drop a Specific String before another string in Python

So I have an application that generally does a good job at collecting information and moving stuff from one AWS s3 bucket to another, and then processing it, but it doesn't really do a good job when people name their file with a pretext string.
Currently, I look for glob:
dump_files = glob.glob('docker-support*.zip')
What this does is I have logic built to only look for things that account for file names that utilize docker-support as the main identifier.
However, I need it to account for times when people do something like
super_Secret123-Production-whatever-docker-support*.zip
Basically, I would like for the function to rename it using that variable dump_files
Should I just set the variable to something like this:
dump_files = glob.glob('*docker-support*.zip')
or
dump_files = glob.glob('/^(.*?)\docker-support*.zip')
The main thing is I am going to want to pick it up, rename it and then strip the part of the file name that is before the actual file name needed for processing: docker-support*.zip as the application needs to look for files in S3 just named in that format.
Code that handles this:
#!/usr/bin/env python3
# main execution loop for dump analysis tool
# Author: Bryce Ryan, Mirantis Inc.
#
# checks for new files in dump_originals, when found, runs run-parts against that file
# v1.1
# pause.main.loop check
# improved error handling
# escape file name to run-parts to avoid metacharacters
#
#
import os
import tempfile
import time
import zipfile
import logging
import shutil
import glob
from datetime import date
import sys
from os import path
logging.basicConfig(filename='/dump/logs/analyzer_logs.txt', level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%Y-%m-%dT%H:%M:%S%z' )
ROOT_DIR = os.path.abspath('..')
logging.debug("ROOT_DIR: {}".format(ROOT_DIR))
DUMP_DIR = os.path.join(ROOT_DIR, 'dump_originals')
logging.debug("DUMP_DIR: {}".format(DUMP_DIR))
WORK_DIR = os.path.join(ROOT_DIR, 'work_dir')
logging.debug("WORK_DIR: {}".format(WORK_DIR))
# can we actually create a file? just because we have perms, or think we do, doesn't mean there are
# enough inodes or capacity to do basic stuff.
with open(os.path.join(DUMP_DIR, "testfile"), 'w'):
pass
logging.info("Beginning event loop for lodestone. Looking for new files in {}".format(DUMP_DIR))
print("Beginning event loop for lodestone.")
sys.stdout.flush()
os.chdir(DUMP_DIR)
logging.basicConfig(filename="analyzer.logs", level=logging.DEBUG)
while True:
# here at the top of the loop, check to see if we should wait for a bit
# typically, because of testing or maintenance
# if the magic, undocumented file exists, wait for 5 sec and check again
# do this forever
while path.exists("/dump/pause.main.loop"):
print("Pausing main loop for 60s, waiting on /dump/pause.main.loop")
time.sleep(60)
dump_files = glob.glob('docker-support*.zip')
try:
if dump_files[0] != '':
logging.debug("files found")
print("================== BEGIN PROCESSING NEW FILE ========= ")
print("File found:", dump_files[0])
print(" ")
logging.info("Processing new file: " + dump_files[0] )
sys.stdout.flush()
support_dump_file = dump_files[0]
# check that it's an actual zip; if not, ignore it
if not zipfile.is_zipfile(support_dump_file):
print("File: " + str(support_dump_file))
print("Inbound file is not recognized as a zip file.\n\n")
logging.info("Inbound file not recognized as a zip file.")
# now move it out of the way so we don't see it again;
# ok if exists on destination and we ignore the error
shutil.move( support_dump_file, "../dump_complete/" )
# no further processing, so back to the top
sys.stdout.flush()
continue
temp_dir = tempfile.mkdtemp(prefix='dump.', dir=WORK_DIR)
os.chmod(temp_dir, 0o777)
logging.info("temp_dir is: " + temp_dir)
# cmd = ROOT_DIR + "/utilities/run-parts --exit-on-error --arg=analyze --arg=" + DUMP_DIR + " --arg=" + support_dump_file + " --arg=" + temp_dir + " " + ROOT_DIR + "/analysis"
cmd = ROOT_DIR + "/utilities/run-parts --arg=analyze --arg=" + DUMP_DIR + " --arg=\'" + support_dump_file + "\' --arg=" + temp_dir + " " + ROOT_DIR + "/analysis"
print(cmd)
logging.info("Will execute: " + cmd )
sys.stdout.flush()
try:
retcode = os.system(cmd)
tempdir =temp_dir
if retcode == 1:
print("Removing temporary work_dir")
logging.debug("Removing temporary work_dir", tempdir)
shutil.rmtree(tempdir, ignore_errors=True)
sys.stdout.flush()
finally:
print("Finally block for cmd. . .")
print("Removing temporary work_dir")
logging.debug("Removing work_dir " + tempdir)
print(tempdir)
sys.stdout.flush()
# shutil.rmtree(tempdir, ignore_errors=True)
os.system('/bin/rm -rf' + tempdir)
sys.stdout.flush()
except:
pass
# pause for a moment; save some processor cycles
sys.stdout.flush()
time.sleep(1)
Right now I do not have the function that will rename this in there.

Print new line not working in Python

The \n just doesn't seem to work for me when I use it with print. I am using Python 2.7.8. I don't get whats wrong, I think \n with a print should print a new line very straight forwardly.
import sys
import os
import subprocess
from collections import OrderedDict
import xmlrpclib
import hawkey
op_name = sys.argv[1]
pkg_name = sys.argv[2]
# Hawkey Configurations
sack = hawkey.Sack()
path = "/home/thejdeep/test_repo/repodata/%s"
repo = hawkey.Repo("test")
repo.repomd_fn = path % "repomd.xml"
repo.primary_fn = path % "b6f6911f7d9fb63f001388f1ecd0766cec060c1d04c703c6a74969eadc24ec97-primary.xml.gz"
repo.filelists_fn = path % "df5897ed6d3f87f2be4432543edf2f58996e5c9e6a7acee054f9dbfe513df4da-filelists.xml.gz"
sack.load_repo(repo,load_filelists=True)
# Main Function
if __name__ == "__main__":
print "Querying the repository\n"
print "-----------------------\n"
print "Found packages :\n"
print "--------------\n"
q = hawkey.Query(sack)
q = q.filter(name=pkg_name,latest_per_arch=True)[0]
if q:
for pkg in q:
print str(pkg)
else:
print "No packages with name "+pkg_name+" found. Exiting"
sys.exit()
print "--------------------"
print "Performing Dependency Check"
Output is something like this. Basically its printing in the same line :
Querying the repository ----------------------- Found packages : --------------

Using print method automatically add \n at the end of the line so its not necessary put \n at the end of each line.

Python subprocess doesn't run until calling process finished

EDIT 1 - added more code
I'm not sure that proc.communicate was needed, it was one of the suggestions I found from some other stackoverflow code.(Sorry I was tired last night and didn't think too much before asking the question.)
I should add that I am not an experienced coder (mechanical engineer) as you can probably tell from my code
In my Gui I have a button to call a subprocess
The subprocess (screenshot-cmd.exe) creates a png of a cropped screen shot but it won't actually produce the file until there is an error or if the button click event is over.
This makes me think that the subprocess is not actually run until the event is finished
I want to call the process several times after a single button press and move the files that it produces after each one is produced
if I use proc.wait(), the process hangs indefinitely.
How do I stop this?
# function to take a single image called 'fileName' and place it in directory 'dir'
def takeImage(dir,fileName):
# calculate the view to capture to get the whole display window in.
clientRect = win32gui.GetClientRect(win32gui.GetForegroundWindow())
windowRect = win32gui.GetWindowRect(win32gui.GetForegroundWindow())
print(windowRect)
windowSize = [windowRect[2]-windowRect[0],windowRect[3]-windowRect[1]]
print(windowSize)
print(clientRect)
diffSize = [windowSize[0] -clientRect[2], windowSize[1] - clientRect[3]]
lrbBorder = diffSize[0]/2
topBorder = diffSize[1] - lrbBorder
print("sizeDiff = " + str(diffSize))
windowName = win32gui.GetWindowText(win32gui.GetForegroundWindow())
handleId = win32gui.GetForegroundWindow()
leftMar = designLabel.GetPosition()[0] + lrbBorder
topMar = designLabel.GetPosition()[1] + topBorder + designLabel.GetSize()[1]
rightMar = leftMar + scene.width
bottMar = topMar+scene.height
margins = [leftMar,topMar,rightMar,bottMar]
print(margins)
# now print the view.
#command_line = r"screenshot-cmd -wt '" + windowName + "' -rc " + str(margins[0]) + " " + str(margins[1]) + " " + str(margins[2]) + " " + str(margins[3]) + " -o " + fileName
command_line = r"screenshot-cmd -wt '" + windowName + "' -rc " + str(margins[0]) + " " + str(margins[1]) + " " + str(margins[2]) + " " + str(margins[3]) + " -o " + fileName
print(command_line)
args = shlex.split(command_line)
proc = subprocess.Popen(args)
proc.wait()
wx.Yield()
if not os.path.isdir(dir):
os.makedirs(dir)
newPath = os.path.join(dir,fileName)
if os.path.exists(newPath):
os.remove(newPath)
oldPath = os.path.join(os.getcwd(), fileName)
print("Old Path: " + oldPath)
print("Exists: " + str(os.path.exists(oldPath)))
shutil.move(oldPath,newPath)
return
#event called upon clicking 'takeTenImag' button
def takeTenImgE(evt):
global designNo
global workingDirectory
global numDesigns
fileNameRoot = "test_"
fileExtention = ".png"
# check there are at least 10 designs
if numDesigns > 9 and os.path.exists(workingDirectory):
# find directory path to put images in
dir = os.path.join(workingDirectory, "images")
# for each design
for x in range(10):
print("design =" + str(designNo))
fileName = fileNameRoot + str(designNo) + fileExtention
print("------------------")
print("for x = " + str(x) + " " + fileName)
# create image and save
print(dir)
takeImage(dir,fileName)
#move to next design
wx.PostEvent(forwardDesign, wx.CommandEvent(wx.wxEVT_COMMAND_BUTTON_CLICKED, forwardDesign.GetId()) )
wx.Yield()
print("design =" + str(designNo))
return
takeTenImg = wx.Button(p, label='Take Ten Images', pos=(rb + visScaleText.GetSize()[0]+10,takeImg.GetPosition()[1]+5 +takeImg.GetSize()[1]), size = (100,30))
takeTenImg.Bind(wx.EVT_BUTTON, takeTenImgE)
https://code.google.com/p/screenshot-cmd/

Barnaby, you may be over-complicating your subprocess use. Popen is typically used for when you need to communicate with the process during the time it is running. From the sound of it, you don't need to do that, so might want to use a higher level function. See the docs on subprocess' various invocations, and perhaps try using the call method. You'll need shell=True, as detailed in SO questions such as this one.

I've found that the error is in my calling of subprocess.
I was using:
command_line = r"screenshot-cmd -wt '" + windowName + ...."
args = shlex.split(command_line)
subprocess.call(args,shell=True)
changing this to:
command_line = r"screenshot-cmd -wt '" + windowName + ...."
subprocess.call(command_line,shell=True)
solves the hang.
What is peculiar is that both options work when not inside a wx button click event(i.e. a python script launched from the command line), but only the second works when inside the wx button click event.
If anyone could enlighten me why that would be most appreciated.
EDIT:
upon further investigation, the hang is caused by trying to specify the active window in screenshot-cmd.
To solve this I find the position of the window using
windowRect = win32gui.GetWindowRect(win32gui.GetForegroundWindow())
and then use screenshot-cmd without specifying a window.
This solves all issues although it is unclear why this causes problems

pass file path through cmd to python script

I am using vb.net and Arcobjects for my program. I am creating a button for ArcMap 10 that will convert a kml to a lyr file.
I am having problems passing variables into the python code. The variables are file paths and it works great if I hard code them in with / instead of . When the variables are dynamically passed in, the program breaks at the "/"s in the path names:
Dim Filelocation As OpenFileDialog = New OpenFileDialog()
Filelocation.Title = "Please point photo of the owner"
Filelocation.InitialDirectory = "B:\GeoSpatialData\Projects\004402 Griffiths\File Structure\Geospatial\GPS\KML"
If Filelocation.ShowDialog = DialogResult.OK Then
Dim kmlFile As String
kmlFile = Filelocation.FileName
Dim args As String
args = kmlFile & " " & kmlFile.Substring(0, kmlFile.LastIndexOf("\")) & " test"
Dim args2 As String = args.Replace("\", "/")
Dim procStartInfo As System.Diagnostics.ProcessStartInfo = New System.Diagnostics.ProcessStartInfo("C:\Python26\python", "C:\Users\KJacobsen\kml_to_shp.py " & args2)
' The following commands are needed to redirect the standard output.
' This means that it will be redirected to the Process.StandardOutput StreamReader.
procStartInfo.RedirectStandardOutput = True
procStartInfo.UseShellExecute = False
' Do not create the black window.
procStartInfo.CreateNoWindow = False
' Now you create a process, assign its ProcessStartInfo, and start it.
Dim proc As New System.Diagnostics.Process()
proc.StartInfo = procStartInfo
proc.Start()
proc.WaitForExit()
' Get the output into a string.
Dim result As String = proc.StandardOutput.ReadToEnd()
' Display the command output.
Console.WriteLine(result)
End If
Catch objException As Exception
' Log the exception and errors.
Console.WriteLine(objException.Message)
End Try
My python script looks like this:
import os
import arcpy
import sys
import glob
arcpy.KMLToLayer_conversion(sys.argv[1],sys.argv[2],sys.argv[3])
print

The path returned contains spaces? It seems so from your initial directory.
In that case the command arguments passed to the script could be wrong.
Try to enclose everything in double quotes and avoid direct manipulation of paths.
Use Path.GetDirectoryName() instead
Dim args As String
args = """" + kmlFile + """ "
args = args & """" & Path.GetDirectoryName(kmlFile) & """ test"

Replace "\" to "\\\"
Does it work?

IOError opening an existing file with Python

Running the following code:
import os
import datetime
import ftplib
currdate = datetime.datetime.now()
formatdate = currdate.strftime("%m-%d-%Y %H%M")
def log():
fqn = os.uname()[1]
ext_ip = urllib2.urlopen('http://whatismyip.org').read()
log = open ('/Users/admin/Documents/locatelog.txt','w')
log.write(str("Asset: %s " % fqn))
log.write(str("Checking in from IP#: %s" % ext_ip))
smush = str(fqn +' # ' + formatdate)
os.rename('/Users/admin/Documents/locatelog.txt','/Users/admin/Documents/%s.txt' % smush )
s = ftplib.FTP('10.7.1.71','username','password')
f = open('/Users/admin/Documents/%s.txt' % smush,'r')
s.storbinary("STOR /Users/admin/Documents/%s.txt" % smush,f)
Generates the following error:
ftplib.error_perm: 550 /Users/admin/Documents/678538.local # 02-24-2010 1301.txt: No such file or directory
I have a feeling something is amiss in this line :
s.storbinary("STOR /Users/admin/Documents/%s.txt" % smush,f)
678538 is the host I am testing on...using Mac OS X 10.5 and Python 2.5.1

Shouldn't it bef = open('/Users/admin/Documents/%s.txt' % smush,'r') ? notice the / in front of Users
If you dont put the first /, the script will think the path to the file is relative to the current directory (where the script is run from)
Edit:
I m not too familiar with Python (I wish) but shouldnt it be:
s.storbinary('STOR /Users/admin/Documents/%s.txt' % smush,f) ?
In your example, Python will treat your string as literal and you want to interpolate the value of smush with %s
Edit 2:
Does the directory /Users/admin/Documents/ exist on your server? If not, I think you will have to create them before copying anything. (Since the error message is about some files/folders missing).
You can create them yourself first. Run your script. If the file is copied successfully, then you can add the creation of the directories from within your script.

remove all spaces from file name .eg in smush = str(fqn +' # ' + formatdate), you are putting a space in front of and after "#". you path looks like
/Users/admin/Documents/something # something
and when you pass it to ftplib, it may have problem. another way is to try putting quotes, eg
s.storbinary("STOR '/Users/admin/Documents/%s.txt'" % smush,f)

Edit:
This version works: Problem was that I was writing to buffer, and not to file.
import os
import urllib2
import datetime
import ftplib
currdate = datetime.datetime.now()
formatdate = currdate.strftime("%m-%d-%Y-%H%M")
def log():
fqn = os.uname()[1]
ext_ip = urllib2.urlopen('http://whatismyip.org').read()
smush = str(fqn + formatdate)
s = ftplib.FTP('10.7.1.71','username','password')
f = open('/Users/admin/Documents/%s.txt' % smush,'w')
f.write(str("Asset: %s " % fqn))
f.write('\n')
f.write(str("Checking in from IP#: %s" % ext_ip))
f.write('\n')
f.write(str("On: %s" % formatdate))
f.close
f = open('/Users/admin/Documents/%s.txt' % smush,'rb')
s.storbinary('STOR %s.txt' % smush , f)
s.close
f.close