Develop Reference

shutil move based on csv filename

hi guys currently moving file based on filename on my csv file but it always move the files first and then read the filename so it always got error already exist like this
Error: Destination path 'Sortir/Membuka kertas contekan/aug1_Data16_133_86.jpg' already exists
CODE
import pandas as pd
data = pd.read_csv('train.csv')
filenames = data['filename'].values
filenames = filenames.tolist()
classes = data['class'].values
classes = classes.tolist()
print(filenames)
print(classes)
import shutil
import os
for index, row in data.iterrows():
print(row['filename'], os.path.join("Sortir",row['class']))
if not os.path.exists(os.path.join("Sortir",row['class'])):
print("[INFO] 'creating {}' directory".format(os.path.join("Sortir",row['class'])))
os.mkdir(os.path.join("Sortir",row['class']))
shutil.move(os.path.join("images",row["filename"]), os.path.join("Sortir",row['class']))
Anyone know how to do the read the row first and then move the file? or maybe keep continue to read other row even if the file that I want to move being already moved?
Found the Answer Code here :
import shutil
import os
import pandas as pd
data = pd.read_csv('test.csv')
filenames = data['filename'].values
filenames = filenames.tolist()
classes = data['class'].values
classes = classes.tolist()
print(filenames)
print(classes)
for index, row in data.iterrows():
if not os.path.exists(os.path.join("SortirTest",row['class'])):
print("[INFO] 'creating {}' directory".format(os.path.join("SortirTest",row['class'])))
os.mkdir(os.path.join("SortirTest",row['class']))
input_name = os.path.join("images", row["filename"])
output_name = os.path.join("SortirTest", row['class'], row['filename'])
if os.path.exists(input_name):
dest = shutil.move(input_name, output_name)
print("This File Has Been Moved:", input_name)
else:
print("This File Doesnt Exist :", input_name)
continue

In shutil.move() function you have to add the filename in the new directory too:
input_name = os.path.join("images", row["filename"])
output_name = os.path.join("Sortir", row['class'], row['filename'])
shutil.move(input_name, output_name)

Have you tried to clear the 'Sortir' folder before running the script?

Incrementing a file name in python

I am making code which generates a new text file with today's date each time it is run. For exemple today's file name would be 2020-10-05. I would like to increment it so that if the program is run one or more times the same day it becomes 2020-10-05_1, _2 etc..
I have this code that I found from another question and i've tried tinkering with it but I'm still stuck. The problem is here they convert the file name to an int 1,2,3 and this way it works but this isn't the result I want.
def incrementfile():
todayday = datetime.datetime.today().date()
output_folder = "//10.2.30.61/c$/Qlikview_Tropal/Raport/"
highest_num = 0
for f in os.listdir(output_folder):
if os.path.isfile(os.path.join(output_folder, f)):
file_name = os.path.splitext(f)[0]
try:
file_num = int(file_name)
if file_num > highest_num:
highest_num = file_num
except ValueError:
print("The file name %s is not an integer. Skipping" % file_name)
output_file = os.path.join(output_folder, str(highest_num + 1) + f"{todayday}" + ".txt")
return output_file
How can I modify this code so that the output I get in the end is something like 2020-10-05_0, _1, _2 etc.. ?
Thanks !

I strongly recommend you to use pathlib instead of os.path.join. This is more convenient.
def incrementfile():
td = datetime.datetime.today().date()
path = pathlib.Path("/tmp") #set your output folder isntead of /tmp
inc = len(list(path.glob(f"{td}*")))+1
outfile = path/f"{td}_{inc}.txt"
return outfile

Not a direct answer to your question, but instead of using _1, _2 etc, you could use a full timestamp with date and current time, which would avoid duplication, EG:
from datetime import datetime
t = str(datetime.now()).replace(":", "-").replace(" ", "_")
print(t)
Example output:
2020-10-05_13-06-53.825870

I think this will work-
import os
import datetime
#assuming files will be .txt format
def incrementfile():
output_folder = "//10.2.30.61/c$/Qlikview_Tropal/Raport/"
files=os.listdir(output_folder)
current_name=datetime.date.today().strftime('%Y-%m-%d_0')
current_num=1
def nameChecker(name,files):
return True if name +'.txt' in files else False
while namChecker(current_name,files):
current_name+='_'+str(current_num)
current_num+=1
return current_name+'.txt'

Issues with python path

I am working on a program in Arcgis however, I have a python issue.
While trying to run the following script I am getting an invalid MXD error which seems to suggest that it cannot find the location of my MXD file.
# modified by ESRI for use as a toolbox script tool
## imported sys to get the raw inputs and os to use for path separator
import arcpy, sys, os
# Set OverWrite if files already exist
arcpy.OverWriteOutput = 1
print "Enter folder path:"
mapDoc = raw_input()
#mapDoc = sys.argv[1]
print os.path.dirname(mapDoc)
# set a variable to the full path and file name of the MXD
fullnam = os.path.basename(mapDoc)
# Strip off the MXD file extension and store as string variable for use in the 'out_pdf'
nam = fullnam.strip(".mxd")
print nam
# Commented this out, since it doesnt need to be a parameter when you use the MXD name as the PDF name
##print "Enter save as name:"
##mapName = sys.argv[2]
map = arcpy.mapping
mxd = map.MapDocument(mapDoc)
map_document = mxd
#out_pdf = r"K:\projects" + "\\" + mapName + ".pdf"
#out_pdf = r"K:\projects" + os.sep + mapName + ".pdf"
#out_pdf = os.path.dirname(mapDoc) + os.sep + mapName + ".pdf"
out_pdf = os.path.dirname(mapDoc) + os.sep + nam + ".pdf"
# Set all the parameters as variables here:
data_frame = 'PAGE_LAYOUT'
resolution = "300"
image_quality = "NORMAL"
colorspace = "RGB"
compress_vectors = "True"
image_compression = "DEFLATE"
picture_symbol = 'RASTERIZE_BITMAP'
convert_markers = "False"
embed_fonts = "True"
layers_attributes = "NONE"
georef_info = "False"
# Due to a known issue, the df_export_width and df_export_height must be set to integers in the code:
map.ExportToPDF(map_document, out_pdf, data_frame, 640, 480, resolution, image_quality, colorspace, compress_vectors, image_compression, picture_symbol, convert_markers, embed_fonts, layers_attributes, georef_info)
# This gives feedback in the script tool dialog:
arcpy.GetMessages()
Now please look at the command prompt.
(arcgispro-py3) C:\Program Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3>"C:\Us
ers\ant\Documents\MyCensus_Files\Python script\Export2PDF.py"
Enter folder path:
C:\Users\ant\Documents\MyCensus_Files\Python script
C:\Users\ant\Documents\MyCensus_Files
Python script
Traceback (most recent call last):
File "C:\Users\ant\Documents\MyCensus_Files\Python script\Export2PDF.py
", line 22, in <module>
mxd = map.MapDocument(mapDoc)
File "C:\Program Files (x86)\ArcGIS\Desktop10.6\ArcPy\arcpy\arcobjects\mixins.
py", line 651, in __init__
assert (os.path.isfile(mxd) or (mxd.lower() == "current")), gp.getIDMessage(
89004, "Invalid MXD filename")
AssertionError: Invalid MXD filename.
As you can see, the path I enter is different from what is being returned by print and I think that is what's causing my issue. Would appreciate it anyone could assist fixing handling this path error or with a python script that accomplishes converting MXD to PDF.
Regards

If you print out mapDoc right before the line map.MapDocument(mapDoc), you would see that you're trying to pass C:\Users\ant\Documents\MyCensus_Files\Python script as a mxd file. That's a directory, not a mxd file.

Try this:
import arcpy, sys, os
arcpy.OverWriteOutput = 1
mapDoc = os.path.join("C:/", "Users", "ant", "Documents", "MyCensus_Files", "Python_script", "test.mxd")
map = arcpy.mapping
mxd = map.MapDocument(mapDoc)
# ...

python code doesn't generate the csv file [closed]

Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 5 years ago.
Improve this question
AIM: I wanted to record all of the files on a variety of hard disk and collects the: file name, folders, and size of the file in megabytes. The code runs, to my knowledge doesn't produce any errors, but doesn't produce the csv file at the end?
What I've tried:
I've tried running the file with Sudo, changing the permissions with chmod +x, checking that python is in the same place for standard user and for sudo user, and lastly removing or commenting out troublesome lines which seem to yield different results or errors depending on OS.
import os
from os import path
import sys
import datetime
from datetime import date, time, timedelta
import time
import csv
#from socket import gethostbyname
#variables
#hostname = str(socket.gethostname())
scandir = "/"
savefiledir = "/Users/joshua/Documents/python/"
textfilename = str(datetime.datetime.now().strftime("%Y-%m-%d")) + "_" "directory_printer.csv"
#change directory to the root directory or the one which you want to scan for files (scandir)
os.getcwd()
os.chdir(scandir)
directory = os.getcwd()
#find all files in a directory and it's sub directory regardless of extension
results = [val for sublist in [[os.path.join(i[0], j) for j in i[2]] for i in os.walk(directory)] for val in sublist]
d = {}
file_count = 0
metadata = []
for file in results:
#full path
try:
fullpath = file
except:
fullpath = None
#file name
try:
file_directory = "/".join(str(file).split('/')[1:-1])
except:
file_directory = None
#file extension
try:
file_ext = str(file).split('/')[-1]
except:
file_ext = None
#subfolders
try:
parts = file_directory.split('/')
sub_folders = ":".join(parts[1:-1])
except:
sub_folders = None
#num subfolders
try:
count_subfolders = len(sub_folders.split(':'))
except:
count_subfolders = None
#filesize megabytes
try:
filesize_mb = os.path.getsize(file)/1024
except:
filesize_mb = None
#date modified
try:
date_modified = datetime.datetime.now() - datetime.datetime.fromtimestamp(path.getmtime(file))
except:
date_modified = None
#time modified
#try:
# time_modified = os.stat(fullpath).st_mtime #time of most recent content modification
#except:
# time_modified = None
#time created (windows)
# try:
# time_created = os.stat(fullpath).st_ctime #platform dependent; time of most recent metadata change on Unix, or the time of creation on Windows)# except:
# time_created = None
#record all file metadata
d[file_count] = {'Full_Path': fullpath, 'File_Directory': file_directory,
'File_Extension': file_ext, 'List_Sub_Folders' : sub_folders,
'Count_Sub_Folders' : count_subfolders, 'Filesize_mb' : filesize_mb,
'Date_Modified' : date_modified}
file_count = file_count + 1
#write the dictinary with the disks file metadata to a csv file
with open(textfilename,'w') as f:
w = csv.writer(f)
w.writerows(d.items())
print("Scanning directory: "
+ str(scandir) + " complete!" + "\n"
+ "The results have been saved to: " + "\n"
+ str(savefiledir)+str(textfilename))

As it is, it looks like your code will write the CSV file to scandir (/), not to savefiledir, because at the beginning of the program you call os.chdir(scandir). If you want to get the file at the right place (where the final printed message says it's saved to) you should do:
# ...
#write the dictinary with the disks file metadata to a csv file
with open(savefiledir + textfilename,'w') as f:
w = csv.writer(f)
w.writerows(d.items())
# ...

Python, open a file when the name is not fully known

I have a list of files with names such as these:
20140911_085234.csv
20140912_040056.csv
What is known is the first part which is the date (the second is a random number). How can I open the correct file if I know the date?
Update: There is one file per day.

As #isedev says, you could use the fnmatch method to find all the files with the "date" pattern. The code could be like this:
from fnmatch import fnmatch
import os
folder_path = '/home/Desktop/project'
all_files = os.listdir(folder_path)
content_file = 'Hello World'
_date = '20140911'
_pattern = _date + '*'
for file_name in all_files:
if fnmatch(file_name, _pattern):
with open(os.path.join(folder_path, file_name), 'wb') as f:
f.write(content_file)
I hope it helps you!

Using glob :
import time
import glob
import os
def open_file_by_date(date):
path = "/path/to/file"
files = glob.glob1(path, date + "_*.csv")
for file in files:
with open(os.path.join(path, file), 'wb') as f:
#do your stuff with file
if __name__ == "__main__":
today = time.strftime("%Y%m%d")
open_file_by_date(today)