How to define folder name when saving JSON file?
I tried to add myfoldername inside open(), but did not work.
Also tried to myfoldername/myfilename in filename definition
Error:
TypeError: an integer is required (got type str)
Code:
import json
# Testing file save
dictionary_data = {"a": 1, "b": 2}
filename = "myfilename" + time.strftime("%Y%m%d-%H%M%S") + ".json"
a_file = open("myfoldername",filename, "w")
json.dump(dictionary_data, a_file)
a_file.close()
This should do the trick.
Use pathlib to manage paths
Create the parent dir if not exist with mkdir
Open the file thanks to the with statement
import json
import time
from pathlib import Path
# Testing file save
dictionary_data = {"a": 1, "b": 2}
filename = Path("myfilename") / Path(f"{time.strftime('%Y%m%d-%H%M%S')}.json")
# create the parent dir if not exist
filename.parent.mkdir(parents=True, exist_ok=True)
with open(filename, "w") as a_file:
json.dump(dictionary_data, a_file)
Related
I want to get all files in a directory (I reached it after doing several for loops - hence fourth.path) that ends with .npy or with csv and then zip those files.
My code is running putting one file only in the zip file. What am I doing wrong?
I tried to change my indents, but no zip file is being created
import json
import os
import zipfile
import zlib
directory = os.path.join(os.getcwd(), 'recs')
radarfolder = 'RadarIfxAvian'
file = os.listdir(directory)
def r(p, name):
p = os.path.join(p, name)
return p.replace("/", "\\")
#This code will list all json files in e ach file
for first in os.scandir(directory):
if first.is_dir():
for second in os.scandir(first.path):
if second.is_dir():
for third in os.scandir(second.path):
if third.is_dir():
radar_folder_name = ''
list_files = ()
for fourth in os.scandir(third.path):
if fourth.is_dir():
if radarfolder in fourth.path:
radar_folder_name = fourth.path
print(radar_folder_name)
list_files = ()
for file in os.listdir(fourth.path):
if file.endswith(".npy") | file.endswith(".csv"):
list_files = (file)
print(list_files)
with zipfile.ZipFile(radar_folder_name +'\\' +'radar.zip', 'w', compression=zipfile.ZIP_DEFLATED ) as zipMe:
zipMe.write(radar_folder_name +'\\' +list_files)
zipMe.close()
I tried to change my indents either resulting in error: TypeError: can only concatenate str (not "tuple") to str or no zip file being created
As I said in my second comment, your problem comes from the 'w' argument in your zipping statement. It causes the zip to be overwritten every time it's opened, which you do for each file you zip in. You can fix this 2 ways (at least):
Replace 'w' with 'a'; this way the files will be appended to your zip (with the side effect that, if you do this several times, files will be added more than once).
Keep the 'w', but only open the zip once, having listed all the files you want to zip before. See my code below.
I've taken the liberty to rewrite the part of your code where you look for the 'RadarIfxAvian' folder, since embedded for are clumsy (and if your folder structure changes, they might not work), replacing it with a multi-purpose recursive function.
Note that the folder structure will be included in the .zip; if you want to zip only the files themselves, consider doing os.chdir(radar_folder_name) before zipping the files.
# This function recursively looks for the 'filename' file or folder
# under 'start_path' and returns the full path, or an empty string if not found.
def find_file(start_path, filename):
if filename in os.listdir(start_path):
return start_path + '/' + filename
for file in os.scandir(start_path):
if not file.is_dir():
continue
if (deep_path:=find_file(start_path + '/' + file.name, filename)):
return deep_path
return ''
directory = os.path.join(os.getcwd(), 'recs')
radarfolder = 'RadarIfxAvian'
radar_folder_name = find_file(directory, radarfolder)
print(radar_folder_name)
list_files = []
for file in os.listdir(radar_folder_name):
if file.endswith(".npy") or file.endswith(".csv"):
list_files.append(file)
with zipfile.ZipFile(radar_folder_name + '/' + 'radar.zip', 'w', compression=zipfile.ZIP_DEFLATED ) as zipMe:
for file in list_files:
zipMe.write(radar_folder_name + '/' + file)
If I understand your code correctly, you are looking for a folder "RadarIfxAvian" and want to place a .ZIP in that folder containing any .CSV or .NPY files in that directory. This should do the equivalent, using os.walk for the recursive search:
import os
import zipfile
for path, dirs, files in os.walk('recs'):
if os.path.basename(path) == 'RadarIfxAvian':
print(path)
with zipfile.ZipFile(os.path.join(path, 'radar.zip'), 'w', zipfile.ZIP_DEFLATED) as zip:
for file in files:
if file.endswith(".npy") | file.endswith(".csv"):
print(file)
zip.write(file)
break # stop search once the directory is found and processed
I adjusted my code with the following steps:
Put the if in a function
writing the the zip by looping over each item in the list I appended
import json
import os
import glob
import zipfile
import zlib
directory = os.path.join(os.getcwd(), 'recs')
radarfolder = 'RadarIfxAvian'
file = os.listdir(directory)
list_files = []
def r(p, name):
p = os.path.join(p, name)
return p.replace("/", "\\")
def tozip(path, file):
filestozip = []
if file.endswith(".npy") or file.endswith(".csv"):
filestozip = (path + '\\' + file)
list_files.append(filestozip)
return list_files
#This code will list all json files in each file
for first in os.scandir(directory):
if first.is_dir():
for second in os.scandir(first.path):
if second.is_dir():
for third in os.scandir(second.path):
if third.is_dir():
radar_folder_name = ''
filestozip = []
list_files.clear()
for fourth in os.scandir(third.path):
if fourth.is_dir():
if radarfolder in fourth.path:
radar_folder_name = fourth.path
for file in os.listdir(fourth.path):
filestozip = tozip(radar_folder_name,file)
print(filestozip)
ZipFile = zipfile.ZipFile(r(radar_folder_name,"radar.zip"), "w")
for a in filestozip:
ZipFile.write(a, compress_type= zipfile.ZIP_DEFLATED)
print(radar_folder_name + "added to zip")
I am trying to generate dag files using python code below.
The code below takes two parameters -
bunch of looped json file input
Template which provides the line which the variables has to be applied
I can successfully create output files but the variables which is replicated from the template file did not change. When the file gets created I want the json variables to be passed to the new file created dynamically.
json file:
{
"DagId": "dag_file_xyz",
"Schedule": "'#daily'",
"Processed_file_name":"xyz1",
"Source_object_name":"'xyz2,}
Template:
processed_file = xyzOperator(
task_id=processed_file_name,
source_bucket=bucket_path,
destination_bucket=destination_bucket,
source_object=source_object_name,
destination_object=destination_object_name,
delimiter='.csv',
move_object=False
Generate file code
import json
import os
import shutil
import fileinput
import ctypes
config_filepath = ('C:\\xyz\\')
dag_template_filename = 'C:\\dagfile\\xyztest.py'
for filename in os.listdir(config_filepath):
print(filename)
f = open(config_filepath + filename)
print(f)
config = json.load(f)
new_filename = 'dags/' + config['DagId'] + '.py'
print(new_filename)
shutil.copyfile(dag_template_filename, new_filename)
for line in fileinput.input(new_filename, inplace=True):
print(line)
line.replace("dag_id", "'" + config['DagId'] + "'"))
line.replace("scheduletoreplace", config['Schedule'])
line.replace("processed_file_name", config['Processed_file_name'])
line.replace("source_object_name", config['Source_object_name'])
line.replace("destination_object_name", config['Destination_object_name'])
print(line, end="")
i have generated image in sequence and save in directory called detected_car folder, then i want to send them to JSON API like https://api.generate_json/, when I get json response from API it's also delete automatic with os.remove("%s" %file), but it's always get that error. this is my code:
import cv2
import os
import json
import requests
import time
car_count = [0]
current_path = os.getcwd()
file = 'detected_car/car' + str(len(car_count)) + ".png"
path = '/home/username/Documents/path/to/image/dir%s' % file
result = []
def save_image(source_image):
cv2.imwrite(current_path + file , source_image)
car_count.insert(0, 1)
with open(path, 'rb') as fp:
response = request.post(
'https://api.generate_json/',
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'KEY_API'})
result.append(response.json())
print(json.dumps(result, indent=2));
with open('data.json', 'w') as outfile:
json.dump(result, outfile)
os.remove("%s" %file)
time.sleep(1)
how to solve this. thankyou.
The file you write out appears to be saved into a different directory than the one you try to remove. This is due to os.getcwd() not returning a trailing /.
When you construct the path you send to cv2.imwrite you concatenate two paths with no / between them so you end up with:
"/a/path/from/getcwd" + "detected_car/car99.png"
Which results in a path that looks like this being sent to cv2.imwrite:
"/a/path/from/getcwddetected_car/car99.png"
But when you go to remove the file, you don't specify the absolute path, and ask os.remove to delete a file that doesn't exist.
I can think of 2 ways of solving this:
Add a trailing slash to current_path:
current_path = os.getcwd() + "/"
Use an absolute path for both cv2.imwrite and os.remove:
current_path = os.getcwd()
file = 'detected_car/car' + str(len(car_count)) + ".png"
temp_file = current_path + file
...
def save_image(source_image):
cv2.imwrite(temp_file , source_image)
...
os.remove("%s" %temp_file)
I want to do the following:
1- Check if a pkl file with a given name exists
2- If not, create a new file with that given name
3- Load the data into that file
if not os.path.isfile(filename):
with open(filename,"wb") as file:
pickle.dump(result, file)
else:
pickle.dump(result, open(filename,"wb") )
However, this rises an error even though I have checked the file exists (shouldnt even enter the if!!) with the given path:
Traceback (most recent call last):
with open(filename_i,"wb") as file:
IsADirectoryError: [Errno 21] Is a directory: '.'
Thanks!
You can do it like this:
import os
import pickle
if not os.path.isfile("test_pkl.pkl"):
with open("test_pkl.pkl",'wb') as file:
pickle.dump("some obejct", file)
So first it checks if file exists, if not create the file ("wb") and then dump some object to it via pickle pickle.dump
Maybe this is more clear:
Imports
import os
import pickle
Create pickle and save data
dict = { 'Test1': 1, 'Test2': 2, 'Test3': 3 }
filename = "test_pkl.pkl"
if not os.path.isfile(filename):
with open(filename,'wb') as file:
pickle.dump(dict, file)
file.close()
Opening the pickle file
infile = open(filename,'rb')
new_dict = pickle.load(infile)
infile.close()
Test the data
print(new_dict)
print(new_dict == dict)
print(type(new_dict))
Output
{'Test1': 1, 'Test2': 2, 'Test3': 3}
True
<class 'dict'>
Im trying to create a script that will read a JSON file and use the variables to select particular folders and files and save them somewhere else.
My JSON is as follows:
{
"source_type": "folder",
"tar_type": "gzip",
"tar_max_age": "10",
"source_include": {"/opt/myapp/config", "/opt/myapp/db, /opt/myapp/randomdata"}
"target_type": "tar.gzip",
"target_path": "/home/user/targetA"
}
So far, I have this Python Code:
import time
import os
import tarfile
import json
source_config = '/opt/myapp/config.JSON'
target_dir = '/home/user/targetA'
def main():
with open('source_config', "r").decode('utf-8') as f:
data = json.loads('source_config')
for f in data["source_include", str]:
full_dir = os.path.join(source, f)
tar = tarfile.open(os.path.join(backup_dir, f+ '.tar.gzip'), 'w:gz')
tar.add(full_dir)
tar.close()
for oldfile in os.listdir(backup_dir):
if str(oldfile.time) < 20:
print("int(oldfile.time)")
My traceback is:
Traceback (most recent call last):
File "/Users/user/Documents/myapp/test/point/test/Test2.py", line 16, in <module>
with open('/opt/myapp/config.json', "r").decode('utf-8') as f:
AttributeError: 'file' object has no attribute 'decode'
How do I fix this?
You are trying to call .decode() directly on the file object. You'd normally do that on the read lines instead. For JSON, however, you don't need to do this. The json library handles this for you.
Use json.load() (no s) to load directly from the file object:
with open(source_config) as f:
data = json.load(f)
Next, you need to address the source_include key with:
for entry in data["source_include"]:
base_filename = os.path.basename(entry)
tar = tarfile.open(os.path.join(backup_dir, base_filename + '.tar.gzip'), 'w:gz')
tar.add(full_dir)
tar.close()
Your JSON also needs to be fixed, so that your source_include is an array, rather than a dictionary:
{
"source_type": "folder",
"tar_type": "gzip",
"tar_max_age": "10",
"source_include": ["/opt/myapp/config", "/opt/myapp/db", "/opt/myapp/randomdata"],
"target_type": "tar.gzip",
"target_path": "/home/user/targetA"
}
Next, you loop over filenames with os.listdir(), which are strings (relative filenames with no path). Strings do not have a .time attribute, if you wanted to read out file timestamps you'll have to use os.stat() calls instead:
for filename in os.listdir(backup_dir):
path = os.path.join(backup_dir, filename)
stats = os.stat(path)
if stats.st_mtime < time.time() - 20:
# file was modified less than 20 seconds ago