My task is to find IfcQuantityArea values of all IfcWall in the project.ifc and export those values to .csv with another attributes such as GlobalId and Name.
The question is how I can "define" the result from def function, so I could set is as variable or list, so I could insert it into a column in my new .csv file?
I tried several ways, but as I print it, it looks fine, but I have no idea how to collect this values to my .csv file. Maybe there is another approach to count the IfcWall areas using some api functions? Any ideas both to python and ifcopenshell environment?
import ifcopenshell
def print_quantities(property_definition):
if 'IfcElementQuantity' == property_definition.is_a():
for quantity in property_definition.Quantities:
if 'IfcQuantityArea' == quantity.is_a():
print('Area value: ' + str(quantity.AreaValue))
model = ifcopenshell.open('D:/.../project-modified.ifc')
products = model.by_type('IfcWall')
for product in products:
if product.IsDefinedBy:
definitions = product.IsDefinedBy
for definition in definitions:
if 'IfcRelDefinesByProperties' == definition.is_a():
property_definition = definition.RelatingPropertyDefinition
print_quantities(property_definition)
if 'IfcRelDefinesByType' == definition.is_a():
type = definition.RelatingType
if type.HasPropertySets:
for property_definition in type.HasPropertySets:
print_quantities(property_definition)
import csv
header = ['GlobalId', 'Name', 'TotalArea']
data = []
for wall in model.by_type('IfcWall'):
row = [wall.GlobalId, wall.Name, AreaValue]
data.append(row)
with open('D:/.../quantities.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
writer.writerows(data)
i have a prblem with importing CSV-file into Database...
Im using SQLAlchemy in Python and wanted to open a CSV-File than show it in QTableWidget to maybe change the values and after write it to DB (New Table).
def setinTable(self):
colcnt = len(self.title)
rowcnt = len(self.data)
self.tabel_model = QtGui.QTableWidget(rowcnt, colcnt)
vheader = QtGui.QHeaderView(QtCore.Qt.Orientation.Vertical)
self.tabel_model.setVerticalHeader(vheader)
hheader = QtGui.QHeaderView(QtCore.Qt.Orientation.Horizontal)
self.tabel_model.setHorizontalHeader(hheader)
self.tabel_model.setHorizontalHeaderLabels(self.title)
for i in range(rowcnt):
for j in range(len(self.data[0])):
item = QtGui.QTableWidgetItem(str(self.data[i][j]))
self.tabel_model.setItem(i, j, item)
self.tabel_model.horizontalHeader().sectionDoubleClicked.connect(self.changeHorizontalHeader)
self.setCentralWidget(self.tabel_model)
Get CSV-Data
def getdata(filepath):
with open(filepath, 'r') as csvfile:
sample = csvfile.read(1024)
dialect = csv.Sniffer().sniff(sample, [';',',','|'])
csvfile.seek(0)
reader = csv.reader(csvfile,dialect=dialect)
header = next(reader)
lines = []
for line in reader:
lines.append(line)
return lines
Reading and showing the CSV-File data in a QTableWidget is working .. but i dont know how to save it to a MySQL Database
For an easier way to load a csv into a database table, check out the 'odo' python project - https://pypi.python.org/pypi/odo/0.3.2
--
To use a table via SQL Alchemy one approach is to use a session and call "update":
myRow = myTable(
column_a = 'foo',
column_b = 'bar')
myRow.column_c = 1 + 2
mySession.update(myRow)
I am trying to read an excel file, extract some data, and write it out as a csv. This is pretty new to me and I'm messing up somewhere: I keep getting an empty csv. I'm sure I'm missing something very basic, but darned if I can see it. Here is the code:
```
import xlrd
import os
import csv
from zipfile import ZipFile
import datetime
datafile = "./2013_ERCOT_Hourly_Load_Data.xls"
outfile = "./2013_Max_Loads.csv"
def parse_file(datafile):
workbook = xlrd.open_workbook(datafile)
sheet = workbook.sheet_by_index(0)
data = None
outputlist = []
for col in range(1, sheet.ncols):
cv = sheet.col_values(col, start_rowx=1, end_rowx=None)
header = sheet.cell_value(0,col)
maxval = max(cv)
maxpos = cv.index(maxval) + 1
maxtime = sheet.cell_value(maxpos, 0)
realtime = xlrd.xldate_as_tuple(maxtime, 0)
year = realtime[0]
month = realtime[1]
day = realtime[2]
hour = realtime[3]
data = [
'Region:', header,
'Year:', year,
'Month:', month,
'Day:', day,
'Hour:', hour,
maxpos,
maxtime,
realtime,
maxval,
]
path = "./2013_Max_Loads.csv"
return outputlist
def save_file(data, filename):
with open(filename, "wb") as f:
writer = csv.writer(f, delimiter='|')
for line in data:
writer.writerow(line)
parse_file(datafile)
save_file(parse_file(datafile),"2013_Max_Loads.csv")
You declare outfile but you don't use it
You aren't passing a directory (path) for the file to be saved in.
I also think that calling parse_file twice might be messing you up. Just pass the filename and call it from within the save_file function.
I also found that you were returning output list as a blank list.
So here, try this. I will assume your xlrd commands are correct, because I have not personally used the module.
import csv
import xlrd
def parse_file(datafile):
workbook = xlrd.open_workbook(datafile)
sheet = workbook.sheet_by_index(0)
outputlist = []
outputlist_append = outputlist.append
for col in range(1, sheet.ncols):
cv = sheet.col_values(col, start_rowx=1, end_rowx=None)
header = sheet.cell_value(0,col)
maxval = max(cv)
maxpos = cv.index(maxval) + 1
maxtime = sheet.cell_value(maxpos, 0)
realtime = xlrd.xldate_as_tuple(maxtime, 0)
year = realtime[0]
month = realtime[1]
day = realtime[2]
hour = realtime[3]
data = [
'Region:', header,
'Year:', year,
'Month:', month,
'Day:', day,
'Hour:', hour,
maxpos,
maxtime,
realtime,
maxval,
]
outputlist_append(data)
return outputlist
def save_file(data, filename):
parse_file(data)
with open(filename, 'wb') as f:
writer = csv.writer(f, delimiter='|')
for line in data:
writer.writerow(line)
return
datafile = "./2013_ERCOT_Hourly_Load_Data.xls"
outfile = "./2013_Max_Loads.csv"
save_file(datafile, outfile)
UPDATE: Edit in code in function save_file() to implement #wwii's suggestion.
Try substituting the new save_file() below:
def save_file(data, filename):
parse_file(data)
with open(filename, 'wb') as f:
wr = csv.writer(f, delimiter='|')
wr.writerows(data)
return
Also, change the variable (you used writer) to something like wr. You really want to avoid any possible conflicts with having a variable with the same name as a method, a function, or class you are calling.
I'm trying to export the attributes of multiple shapefiles all contained in one folder to a text file. I wrote the code below to do so but it is only exporting the file names to a text file. Any ideas on what I may be doing wrong? I've been troubleshooting and researching for a while.
import arcpy
from arcpy import env
arcpy.env.workspace = "C:\\user\\rainfiles"
table = "C:\\user\\rain_files"
outWorkspace = "C:\\user"
fclist = arcpy.ListFields(table)
field_names = [field.name for field in fclist]
for field in fclist:
with open(r'C:\\user\\Exports.txt', 'w') as f:
for field in fclist:
f.write(field + '\n')
with open(r'C:\\user\\Exports.txt', 'r') as f:
w = csv.writer(f)
w.writerow(field_names)
for row in arcpy.SearchCursor(table):
field_vals = [row.getValue(field.name) for field in fclist]
w.writerow(field_vals)
del row
Here's one way:
import arcpy
import csv
f = open(r'C:\\user\\Exports.txt', 'w')
w = csv.writer(f, lineterminator='\n')
arcpy.env.workspace = "C:\\user\\rainfiles"
shapefileList = arcpy.ListFeatureClasses("*.shp")
for table in shapefileList:
f.write("Shapefile:\n")
f.write(table + "\n")
fieldList = arcpy.ListFields(table)
field_names = [field.name for field in fieldList]
w.writerow(field_names)
for row in arcpy.SearchCursor(table):
field_vals = []
for field in fieldList:
val = row.getValue(field.name)
# See if it's a geometry field; if so, use WKT
try:
val = val.WKT
except AttributeError:
# It's not a geometry, and that's okay
pass
field_vals.append(val)
w.writerow(field_vals)
Here's my code, really simple stuff...
import csv
import json
csvfile = open('file.csv', 'r')
jsonfile = open('file.json', 'w')
fieldnames = ("FirstName","LastName","IDNumber","Message")
reader = csv.DictReader( csvfile, fieldnames)
out = json.dumps( [ row for row in reader ] )
jsonfile.write(out)
Declare some field names, the reader uses CSV to read the file, and the filed names to dump the file to a JSON format. Here's the problem...
Each record in the CSV file is on a different row. I want the JSON output to be the same way. The problem is it dumps it all on one giant, long line.
I've tried using something like for line in csvfile: and then running my code below that with reader = csv.DictReader( line, fieldnames) which loops through each line, but it does the entire file on one line, then loops through the entire file on another line... continues until it runs out of lines.
Any suggestions for correcting this?
Edit: To clarify, currently I have: (every record on line 1)
[{"FirstName":"John","LastName":"Doe","IDNumber":"123","Message":"None"},{"FirstName":"George","LastName":"Washington","IDNumber":"001","Message":"Something"}]
What I'm looking for: (2 records on 2 lines)
{"FirstName":"John","LastName":"Doe","IDNumber":"123","Message":"None"}
{"FirstName":"George","LastName":"Washington","IDNumber":"001","Message":"Something"}
Not each individual field indented/on a separate line, but each record on it's own line.
Some sample input.
"John","Doe","001","Message1"
"George","Washington","002","Message2"
The problem with your desired output is that it is not valid json document,; it's a stream of json documents!
That's okay, if its what you need, but that means that for each document you want in your output, you'll have to call json.dumps.
Since the newline you want separating your documents is not contained in those documents, you're on the hook for supplying it yourself. So we just need to pull the loop out of the call to json.dump and interpose newlines for each document written.
import csv
import json
csvfile = open('file.csv', 'r')
jsonfile = open('file.json', 'w')
fieldnames = ("FirstName","LastName","IDNumber","Message")
reader = csv.DictReader( csvfile, fieldnames)
for row in reader:
json.dump(row, jsonfile)
jsonfile.write('\n')
You can use Pandas DataFrame to achieve this, with the following Example:
import pandas as pd
csv_file = pd.DataFrame(pd.read_csv("path/to/file.csv", sep = ",", header = 0, index_col = False))
csv_file.to_json("/path/to/new/file.json", orient = "records", date_format = "epoch", double_precision = 10, force_ascii = True, date_unit = "ms", default_handler = None)
import csv
import json
file = 'csv_file_name.csv'
json_file = 'output_file_name.json'
#Read CSV File
def read_CSV(file, json_file):
csv_rows = []
with open(file) as csvfile:
reader = csv.DictReader(csvfile)
field = reader.fieldnames
for row in reader:
csv_rows.extend([{field[i]:row[field[i]] for i in range(len(field))}])
convert_write_json(csv_rows, json_file)
#Convert csv data into json
def convert_write_json(data, json_file):
with open(json_file, "w") as f:
f.write(json.dumps(data, sort_keys=False, indent=4, separators=(',', ': '))) #for pretty
f.write(json.dumps(data))
read_CSV(file,json_file)
Documentation of json.dumps()
I took #SingleNegationElimination's response and simplified it into a three-liner that can be used in a pipeline:
import csv
import json
import sys
for row in csv.DictReader(sys.stdin):
json.dump(row, sys.stdout)
sys.stdout.write('\n')
You can try this
import csvmapper
# how does the object look
mapper = csvmapper.DictMapper([
[
{ 'name' : 'FirstName'},
{ 'name' : 'LastName' },
{ 'name' : 'IDNumber', 'type':'int' },
{ 'name' : 'Messages' }
]
])
# parser instance
parser = csvmapper.CSVParser('sample.csv', mapper)
# conversion service
converter = csvmapper.JSONConverter(parser)
print converter.doConvert(pretty=True)
Edit:
Simpler approach
import csvmapper
fields = ('FirstName', 'LastName', 'IDNumber', 'Messages')
parser = CSVParser('sample.csv', csvmapper.FieldMapper(fields))
converter = csvmapper.JSONConverter(parser)
print converter.doConvert(pretty=True)
I see this is old but I needed the code from SingleNegationElimination however I had issue with the data containing non utf-8 characters. These appeared in fields I was not overly concerned with so I chose to ignore them. However that took some effort. I am new to python so with some trial and error I got it to work. The code is a copy of SingleNegationElimination with the extra handling of utf-8. I tried to do it with https://docs.python.org/2.7/library/csv.html but in the end gave up. The below code worked.
import csv, json
csvfile = open('file.csv', 'r')
jsonfile = open('file.json', 'w')
fieldnames = ("Scope","Comment","OOS Code","In RMF","Code","Status","Name","Sub Code","CAT","LOB","Description","Owner","Manager","Platform Owner")
reader = csv.DictReader(csvfile , fieldnames)
code = ''
for row in reader:
try:
print('+' + row['Code'])
for key in row:
row[key] = row[key].decode('utf-8', 'ignore').encode('utf-8')
json.dump(row, jsonfile)
jsonfile.write('\n')
except:
print('-' + row['Code'])
raise
Add the indent parameter to json.dumps
data = {'this': ['has', 'some', 'things'],
'in': {'it': 'with', 'some': 'more'}}
print(json.dumps(data, indent=4))
Also note that, you can simply use json.dump with the open jsonfile:
json.dump(data, jsonfile)
Use pandas and the json library:
import pandas as pd
import json
filepath = "inputfile.csv"
output_path = "outputfile.json"
df = pd.read_csv(filepath)
# Create a multiline json
json_list = json.loads(df.to_json(orient = "records"))
with open(output_path, 'w') as f:
for item in json_list:
f.write("%s\n" % item)
How about using Pandas to read the csv file into a DataFrame (pd.read_csv), then manipulating the columns if you want (dropping them or updating values) and finally converting the DataFrame back to JSON (pd.DataFrame.to_json).
Note: I haven't checked how efficient this will be but this is definitely one of the easiest ways to manipulate and convert a large csv to json.
As slight improvement to #MONTYHS answer, iterating through a tup of fieldnames:
import csv
import json
csvfilename = 'filename.csv'
jsonfilename = csvfilename.split('.')[0] + '.json'
csvfile = open(csvfilename, 'r')
jsonfile = open(jsonfilename, 'w')
reader = csv.DictReader(csvfile)
fieldnames = ('FirstName', 'LastName', 'IDNumber', 'Message')
output = []
for each in reader:
row = {}
for field in fieldnames:
row[field] = each[field]
output.append(row)
json.dump(output, jsonfile, indent=2, sort_keys=True)
def read():
noOfElem = 200 # no of data you want to import
csv_file_name = "hashtag_donaldtrump.csv" # csv file name
json_file_name = "hashtag_donaldtrump.json" # json file name
with open(csv_file_name, mode='r') as csv_file:
csv_reader = csv.DictReader(csv_file)
with open(json_file_name, 'w') as json_file:
i = 0
json_file.write("[")
for row in csv_reader:
i = i + 1
if i == noOfElem:
json_file.write("]")
return
json_file.write(json.dumps(row))
if i != noOfElem - 1:
json_file.write(",")
Change the above three parameter, everything will be done.
import csv
import json
csvfile = csv.DictReader('filename.csv', 'r'))
output =[]
for each in csvfile:
row ={}
row['FirstName'] = each['FirstName']
row['LastName'] = each['LastName']
row['IDNumber'] = each ['IDNumber']
row['Message'] = each['Message']
output.append(row)
json.dump(output,open('filename.json','w'),indent=4,sort_keys=False)