import subprocess import os import shutil from tqdm import tqdm import time def free_up_space(path): cmd = f'attrib +U -P /S /D "{path}"' print(cmd) subprocess.run(cmd, capture_output=True, text=True) def copy_file(src_root, dst_root, file_path): # Copy files to the relative paths in the dst folder, always input root of src rel_path = os.path.relpath(file_path, src_root) dst_dir = os.path.join(dst_root, os.path.dirname(rel_path)) os.makedirs(dst_dir, exist_ok=True) shutil.copy2(file_path, dst_dir) def log_copied_file(log_path, file_path): # Append the copied file's path to the log. with open(log_path, 'a') as log: log.write(file_path + '\n') def is_file_logged(log_path, file_path): # Check if the file's path is in the log. if not os.path.exists(log_path): return False with open(log_path, 'r') as log: return file_path in log.read() def process_files_NAS(df, src_root, dst_root, log_path): total_size = 0 error_list = [] for _, row in tqdm(df[["File Size (MB)", "File_Path"]].iterrows(), total=len(df)): total_size = total_size + row["File Size (MB)"] # Check if file has already been copied try: if is_file_logged(log_path, row["File_Path"]): continue # skip copying this file if total_size >= 5000: free_up_space(dst_root + "*") print("Freeing up Space ...") time.sleep(10) total_size = 0 except Exception: print("Error on file") continue try: copy_file(src_root, dst_root, file_path=row["File_Path"]) # Log the successfully copied file log_copied_file(log_path, row["File_Path"]) except FileNotFoundError: error_list.append(row["File_Path"]+","+"Not Found") except PermissionError: error_list.append(row["File_Path"]+","+"Permission Error") except Exception: error_list.append(row["File_Path"]+","+"Unknown Error") free_up_space(dst_root + "*")