Develop Reference

Python compare filename with folder name

I just start python and i have to compare filename with folder name to launch the good sh script. (i'm using airflow)
import glob
import os
import shutil
from os import path
odsPath = '/apps/data/02_ODS/'
receiptPath = '/apps/data/80_DATA/01_Receipt/'
for files in os.listdir(receiptPath):
if(files.startswith('MEM_ZMII') or files.startswith('FMS') and files.endswith('.csv')):
parsedFiles = files.split('_')
pattern = '_'.join(parsedFiles[0:2])
fileName = '_'.join(parsedFiles[2:5])
fileName = fileName.split('-')[0].lower()
# print('appCode: ', pattern)
# print('fileName: ', fileName)
for odsFolder in os.listdir(odsPath):
if(odsFolder == fileName):
print('it exist: ', str(fileName))
else:
print('it\'s not')
I got 3 files in receiptPath , it only matching for 1 file, but not the others. Can someone help me?
Thank a lot!

Ok, your problem is that you overwrite your variable fileName, so at the end of the first for loop, it only keeps the last value, which is material_makt. The solution consists in saving all the filenames in a list fileNames_list, and then you can check if (odsFolder in fileNames_list) :
import glob
import os
import shutil
from os import path
odsPath = '/apps/data/02_ODS/'
receiptPath = '/apps/data/80_DATA/01_Receipt/'
fileNames_list = []
for files in os.listdir(receiptPath):
if(files.startswith('MEM_ZMII') or files.startswith('FMS') and files.endswith('.csv')):
parsedFiles = files.split('_')
pattern = '_'.join(parsedFiles[0:2])
fileName = '_'.join(parsedFiles[2:5])
fileName = fileName.split('-')[0].lower()
fileNames_list.append(fileName)
for odsFolder in os.listdir(odsPath):
if (odsFolder in fileNames_list):
print('it exist:', str(odsFolder))
else:
print('it\'s not')
Output :
it exist: zcormm_familymc
it exist: kpi_obj_data
it exist: material_makt

Python: How to delete the folders called trash that are older than 3 months

I'm looking to create a script that will look through specific directories and its subdirectories, and if it contains a folder called trash, and it is older than 3 months it will delete that trash folder and its contents.
The code i have already tried is shown below. It contains a couple of options of how to get the time of 3 months. I have also used os.walk to try to traverse the directories, but i have not managed to target just the trash folders and then remove them.
import os, sys, time
from datetime import datetime
from datetime import timedelta
from subprocess import call
import path
now = time.time()
cutoff = now - (30)#(91 * 86400)
three_months = datetime.today() - timedelta(0,0,0,0,0,1)
path = '/users/shoot_station_5/documents/folderfordeletetest/'
for root, d_names, f_names in os.walk(path):
print (root, d_names, f_names)
for _dir in d_names:
time_thing = os.path.getmtime(os.path.join (root, _dir))
time_thing1 = datetime.fromtimestamp(time_thing)
if time_thing1 <= three_months and _dir == 'trash':
# fname = os.path.join(d_names, _dir)
# if time.ctime(os.path.getmtime(os.path.join (root, _dir))) < cutoff:
print ('good')

Below code delete a directory called trash that are older than 3 months and all its contents.
import os
import datetime
import shutil
def Delete_Folder(filesPath,No_of_Days):
for root, d_names, f_names in os.walk(filesPath):
for _dir in d_names:
if str(_dir).lower() == "trash":
DName = os.path.join (root, _dir)
today = datetime.datetime.today()
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(DName))
duration = today - modified_date
if (duration.days > No_of_Days):
shutil.rmtree(DName)
# Call Function
NoDays = 90
filesPath = r"C:\Sample"
Delete_Folder(filesPath,NoDays)

How to extract the file name from a file path?

I have the following code:
os.listdir("staging")
# Seperate filename from extension
sep = os.sep
# Change the casing
for n in os.listdir("staging"):
print(n)
if os.path.isfile("staging" + sep + n):
filename_one, extension = os.path.splitext(n)
os.rename("staging" + sep + n, "staging" + sep + filename_one.lower() + extension)
# Show the new file names
print ('\n--------------------------------\n')
for n in os.listdir("staging"):
print (n)
# Remove the blanks, -, %, and /
for n in os.listdir("staging"):
print (n)
if os.path.isfile("staging" + sep + n):
filename_zero, extension = os.path.splitext(n)
os.rename("staging" + sep + n , "staging" + sep + filename_zero.replace(' ','_').replace('-','_').replace('%','pct').replace('/','_') + extension)
# Show the new file names
print ('\n--------------------------------\n')
for n in os.listdir("staging"):
print (n)
"""
In order to fix all of the column headers and to solve the encoding issues and remove nulls,
first read in all of the CSV's to python as dataframes, then make changes and rewrite the old files
"""
import os
import glob
import pandas as pd
files = glob.glob(os.path.join("staging" + "/*.csv"))
print(files)
# Create an empty dictionary to hold the dataframes from csvs
dict_ = {}
# Write the files into the dictionary
for file in files:
dict_[file] = pd.read_csv(file, header = 0, dtype = str, encoding = 'cp1252').fillna('')
In the dictionary, the dataframes are named as "folder/name(csv)" what I would like to do is remove the prefix "staging/" from the keys in the dictionary.
How can I do this?

If all you want to do is truncate the file paths to just the filename, you can use os.path.basename:
for file in files:
fname = os.path.basename(file)
dict_[fname] = (pd.read_csv(file, header=0, dtype=str, encoding='cp1252')
.fillna(''))
Example:
os.path.basename('Desktop/test.txt')
# 'test.txt'

import os
pathname ='c:\\hello\\dickins\\myfile.py'
head, tail = os.path.split(pathname)
print head
print tail

This article here worked out just fine for me
import os
inputFilepath = 'path/to/file/foobar.txt'
filename_w_ext = os.path.basename(inputFilepath)
filename, file_extension = os.path.splitext(filename_w_ext)
#filename = foobar
#file_extension = .txt
path, filename = os.path.split(path/to/file/foobar.txt)
# path = path/to/file
# filename = foobar.txt
Hope it helps someone searching for this answer

In the same spirt as truncate the file paths, use pathlib in python standard library. It will turn the path into an easy to use class.
from pathlib import Path
path = Path('Desktop/folder/test.txt')
path.name # test.txt
path.stem # test
path.suffix # .txt
path.parent.name # folder
path.parent.name.name # Desktop

As ColdSpeed said, you can use "os.path.basename" to truncate a file to its name, but I think what you are refering to is the ability to pycache the data?
For Example here is my Directory:
You see the pycache folder? that initializes it as a module.
Then, you can import a file from that module (for example the staging.txt file and operate on it.)
I use the IpConfig.txt File from the assets folder level (or should be) and take a line of information out of it.
import pygame as pyg
import sys
import os
import math
import ssl
import socket as sock
import ipaddress as ipad
import threading
import random
print("Modules Installed!")
class two:
# Find out how to refer to class super construct
def main(Display, SecSock, ipadd, clock):
# I have code here that has nothing to do with the question...
def __init__():
print("Initializing[2]...")
# Initialization of Pygame and SSL Socket goes here
searchQuery = open("IpConfig.txt", 'r') #Opening the File IpConfig(Which now should open on the top level of the game files)
step2 = searchQuery.readlines()# read the file
ipadd = step2[6] # This is what you should have or something similar where you reference the line you want to copy or manipulate.
main(gameDisplay, SSLSock, ipadd, clock)# Im having issues here myself - (main() is not defined it says)
print(ipadd)
print("Server Certificate Configuration Enabled...")
__init__() # Start up the procedure

How to batch process a folder of videos using MoviePy

I wrote a MoviePy script that takes an input video, does some processing, and outputs a video file. I want to run this through an entire folder of videos. Any help or direction is appreciated.
Here's what I tried...
for f in *; do python resize.py $f; done
and resize.py source code here:
from moviepy.editor import *
clip = VideoFileClip(input)
clip1 = clip.rotate(270)
clip2 = clip1.crop(x_center=540,y_center=960,width=1080,height=608)
clip3 = clip2.resize(width=1920)
clip3.write_videofile(output,codec='libx264')
Really wasn't sure what to put for "input" and "output" in my .py file.
Thanks,
Evan

I know you have an answer on Github, but I'll add my own solution.
First, you'll want to put your code inside a function:
def process_video(input):
"""Parameter input should be a string with the full path for a video"""
clip = VideoFileClip(input, output)
clip1 = clip.rotate(270)
clip2 = clip1.crop(x_center=540,y_center=960,width=1080,height=608)
clip3 = clip2.resize(width=1920)
clip3.write_videofile(output,codec='libx264')
Then, you can have a function that returns a list of file paths, and a list of final file names to use with the above function (note that the final file names will be the same as the original file names but with "output" in front):
import os
def get_video_paths(folder_path):
"""
Parameter folder_path should look like "Users/documents/folder1/"
Returns a list of complete paths
"""
file_name_list = os.listdir(folder_path)
path_name_list = []
final_name_list = []
for name in file_name_list:
# Put any sanity checks here, e.g.:
if name == ".DS_Store":
pass
else:
path_name_list.append(folder_path + name)
# Change the format of the output file names below
final_name_list.append(folder_path + "output" + name)
return path_name_list, final_name_list
Finally, at the bottom, we get the input folder, and utilise the above two functions:
if __name__ == "__main__":
video_folder = input("What folder would you like to process? ")
path_list, final_name_list = get_video_paths(video_folder)
for path, name in zip(path_list, final_name_list):
process_video(path, name)
print("Finished")
Just watch out, because this will crash if there are any files in the folder that can't be read as a movie. For instance, on mac, the OS puts a ".DS_Store" file in each folder, which will crash the program. I've put an area for a sanity check to ignore certain filenames.
Complete code:
import os
from moviepy.editor import *
def process_video(input, output):
"""Parameter input should be a string with the full path for a video"""
clip = VideoFileClip(input)
clip1 = clip.rotate(270)
clip2 = clip1.crop(x_center=540,y_center=960,width=1080,height=608)
clip3 = clip2.resize(width=1920)
clip3.write_videofile(output,codec='libx264')
def get_video_paths(folder_path):
"""
Parameter folder_path should look like "Users/documents/folder1/"
Returns a list of complete paths
"""
file_name_list = os.listdir(folder_path)
path_name_list = []
final_name_list = []
for name in file_name_list:
# Put any sanity checks here, e.g.:
if name == ".DS_Store":
pass
else:
path_name_list.append(folder_path + name)
final_name_list.append(folder_path + "output" + name)
return path_name_list, final_name_list
if __name__ == "__main__":
video_folder = input("What folder would you like to process? ")
path_list, final_name_list = get_video_paths(video_folder)
for path, name in zip(path_list, final_name_list):
process_video(path, name)
print("Finished")

I responded on your Github issue #542, but I copied it here for future reference!
First off, the below example isn't ironclad, but it should do what you need.
You can achieve this via something like this:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Convert all media assets located in a specified directory."""
import glob
import os
from optparse import OptionParser
from moviepy.editor import VideoFileClip
def get_dir_files(dir_path, patterns=None):
"""Get all absolute paths for pattern matched files in a directory.
Args:
dir_path (str): The path to of the directory containing media assets.
patterns (list of str): The list of patterns/file extensions to match.
Returns:
(list of str): A list of all pattern-matched files in a directory.
"""
if not patterns or type(patterns) != list:
print('No patterns list passed to get_dir_files, defaulting to patterns.')
patterns = ['*.mp4', '*.avi', '*.mov', '*.flv']
files = []
for pattern in patterns:
dir_path = os.path.abspath(dir_path) + '/' + pattern
files.extend(glob.glob(dir_path))
return files
def modify_clip(path, output):
"""Handle conversion of a video file.
Args:
path (str): The path to the directory of video files to be converted.
output (str): The filename to associate with the converted file.
"""
clip = VideoFileClip(path)
clip = clip.rotate(270)
clip = clip.crop(x_center=540, y_center=960, width=1080, height=608)
clip = clip.resize(width=1920)
clip.write_videofile(output, codec='libx264')
print('File: {} should have been created.'.format(output))
if __name__ == '__main__':
status = 'Failed!'
parser = OptionParser(version='%prog 1.0.0')
parser.add_option('-p', '--path', action='store', dest='dir_path',
default='.', type='string',
help='the path of the directory of assets, defaults to .')
options, args = parser.parse_args()
print('Running against directory path: {}'.format(options.dir_path))
path_correct = raw_input('Is that correct?').lower()
if path_correct.startswith('y'):
dir_paths = get_dir_files(options.dir_path)
for dir_path in dir_paths:
output_filename = 'converted_' + os.path.basename(dir_path)
modify_clip(path=dir_path, output=output_filename)
status = 'Successful!'
print('Conversion {}'.format(status))
With the above example, you can simply drop that into the directory of assets you wish to convert and run: python this_file.py and it should convert the files for you in the same directory with the name prepended with: converted_
Likewise, you can drop that file anywhere and run it against an absolute path:
python this_file.py -p /Users/thisguy/media and it will convert all files with the extensions: ['*.mp4', '*.avi', '*.mov', '*.flv']
Either way, let me know if you have any questions (or if this resolves your issue) and I'll do my best to help you out!
Thanks for using moviepy!

Run file inside a zipfile?

Is it possible to run a .html or .exe for example, that is inside a zipfile? I'm using the Zipfile module.
Here's my sample code:
import zipfile
z = zipfile.ZipFile("c:\\test\\test.zip", "r")
x = ""
g = ""
for filename in z.namelist():
#print filename
y = len(filename)
x = str(filename)[y - 5:]
if x == ".html":
g = filename
f = z.open(g)
After f = z.open(g), I don't know what to do next. I tried using the .read() but it only reads whats inside of the html, what I need is for it to run or execute.
Or is there any othere similar ways to do this?

The best approach will be to extract the required file to the Windows temp directory and execute it. I have modified your original code to create a temp file and execute it:
import zipfile
import shutil
import os
z = zipfile.ZipFile("c:\\test\\test.zip", "r")
x = ""
g = ""
basename = ""
for filename in z.namelist():
print filename
y = len(filename)
x = str(filename)[y - 5:]
if x == ".html":
basename = os.path.basename(filename) #get the file name and extension from the return path
g = filename
print basename
break #found what was needed, no need to run the loop again
f = z.open(g)
temp = os.path.join(os.environ['temp'], basename) #create temp file name
tempfile = open(temp, "wb")
shutil.copyfileobj(f, tempfile) #copy unzipped file to Windows 'temp' folder
tempfile.close()
f.close()
os.system(temp) #run the file

Run the first .html file in a zip archive specified at the command line:
#!/usr/bin/env python
import os
import shutil
import sys
import tempfile
import webbrowser
import zipfile
from subprocess import check_call
from threading import Timer
with zipfile.ZipFile(sys.argv[1], 'r') as z:
# find the first html file in the archive
member = next(m for m in z.infolist() if m.filename.endswith('.html'))
# create temporary directory to extract the file to
tmpdir = tempfile.mkdtemp()
# remove tmpdir in 5 minutes
t = Timer(300, shutil.rmtree, args=[tmpdir], kwargs=dict(ignore_errors=True))
t.start()
# extract the file
z.extract(member, path=tmpdir)
filename = os.path.join(tmpdir, member.filename)
# run the file
if filename.endswith('.exe'):
check_call([filename]) # run as a program; wait it to complete
else: # open document using default browser
webbrowser.open_new_tab(filename) #NOTE: returns immediately
Example
T:\> open-from-zip.py file.zip
As an alternative to webbrowser you could use os.startfile(os.path.normpath(filename)) on Windows.