Develop Reference

Split json file into multiple csv files depending on date?

I am trying to split up a json file from alpha-vantages api into separate files depending on the date. I'm also trying to reformat the file to have blank values in the gaps where dates are missing. The following code is what I have come up with but it gives me the TypeError: 'list' object is not callable". I'm fairly new to python and pandas so I'm sure there is a better way to go about this.
import requests
import pandas as pd
from datetime import datetime, timedelta
from dateutil import parser
import numpy as np
from pandas import DataFrame
import json
symbol = "MSFT"
symbol_list = symbol.split(",")
def num_el(list):
count = 0
for element in list:
count += 1
return count
def csv_make(sy, dar, dat):
csv_file = open(f"{sy}_1min_{dar}.csv", "w", newline="")
csv_file.write(dat)
csv_file.close()
i = 0
x = -1
n = num_el(symbol_list)
while i < n:
namesym = symbol_list[x]
ticker = namesym
api_key = 'APIKEYHERE'
url = f'https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={ticker}&outputsize=full&interval=1min&apikey={api_key}'
data = requests.get(url)
dsf = data.json()
daf = pd.DataFrame(dsf['Time Series (1min)'])
dxf: DataFrame = daf.T
dxf.index.name = 'time'
dxf.reset_index(inplace=True)
dxf['time'] = pd.to_datetime(dxf['time'])
dxf['minute'] = dxf['time'].dt.time
dxf['day'] = dxf['time'].dt.day
dxf['date'] = dxf['time'].dt.date
agg = dxf.groupby([dxf['day']])
length1 = dxf.groupby([dxf['day']]).size()
length = pd.DataFrame(length1)
length.index.name = 'day'
length.reset_index(inplace=True)
length_sum = length[0].sum()
v = 0
d = length_sum
b = len(length)
x2 = length_sum
while v < b:
a = length[0][v]
x2 -= length[0][v]
xd = agg.get_group(length['day'][v])
date = xd['date'][x2]
max_dt = parser.parse(str(max(xd['minute'])))
min_dt = parser.parse(str(min(xd['minute'])))
dt_range = []
while min_dt <= max_dt:
dt_range.append(min_dt.strftime("%H:%M:%S"))
min_dt += timedelta(seconds=60)
complete_df = pd.DataFrame({'minute': dt_range})
xy = complete_df.astype('str')
yx = xd.astype('str')
dasf = xy.merge(yx, how='left', on='minute')
dasf['ev'] = np.where(dasf['1. open'].notnull(), 'False', 'True')
time = []
open = []
high = []
low = []
close = []
volume = []
empty_value = []
for ib in range(len(dasf)):
time.append(dasf['minute'][ib])
open.append(dasf['1. open'][ib])
high.append(dasf['2. high'][ib])
low.append(dasf['3. low'][ib])
close.append(dasf['4. close'][ib])
volume.append(dasf['5. volume'][ib])
empty_value.append(dasf['ev'][ib])
time_df = pd.DataFrame(time).rename(columns={0: 'Time'})
open_df = pd.DataFrame(open).rename(columns={0: 'Open'})
high_df = pd.DataFrame(high).rename(columns={0: 'High'})
low_df = pd.DataFrame(low).rename(columns={0: 'Low'})
close_df = pd.DataFrame(close).rename(columns={0: 'Close'})
volume_df = pd.DataFrame(volume).rename(columns={0: 'Volume'})
empty_value_df = pd.DataFrame(empty_value).rename(columns={0: 'Empty Value'})
frames = [time_df, open_df, high_df, low_df, close_df, volume_df, empty_value_df]
df = pd.concat(frames, axis=1, join='inner')
df = df.set_index('Time')
ad = df.to_csv()
csv_make(namesym, date, ad)
v += 1
i += 1

Change Trendline-Colour with openPyxl

I'm trying to write a Script in Python with openpyxl, that saves data in a excel-file and also draws a scatterchart with some trendlines. The problem with openpyxl is, that trendlines in default-mode are always colored black, and so my questione is, how to color them in different colors.
from openpyxl.chart import *
from openpyxl.chart.trendline import Trendline
from openpyxl.styles import *
from openpyxl.chart.shapes import *
from openpyxl.drawing.colors import *
wb = Workbook()
ws = wb.create_sheet(name)
#writes the Headlines of the columns into excel-file
for c in range(1,5):
ws.cell(1, c).value = ueberschriften[c-1]
#writes the data into excel-file
for c in range(2, points+2):
e = c-2
for d in range(1,5):
f = d-1
b = a[e][f]
ws.cell(c, d).value = b
for row in a:
ws.append(row)
chart = ScatterChart()
chart.title = "Measuring"
chart.style = 13
chart.x_axis.title = 'Time'
chart.y_axis.title = 'Value'
xvalues = Reference(ws, min_col=1, min_row=2, max_row=points+1)
for i in range(2, 5):
values = Reference(ws, min_col=i, min_row=1, max_row=points+1)
series = Series(values, xvalues, title_from_data=True)
chart.series.append(series)
l = chart.series[0]
l.graphicalProperties.line.solidFill = "FF0000"
l.trindline = Trendline(trendlineType = 'poly', order = fit_order) #trendline with polinomial fitting
l1 = chart.series[1]
l1.graphicalProperties.line.solidFill = "0000FF"
line1.trendline = Trendline(trendlineType = 'poly', order = fit_order)
l2 = chart.series[2]
l2.graphicalProperties.line.solidFill = "00FF00"
l2.trendline = Trendline(trendlineType = 'poly', order = fit_order)
ws.add_chart(chart, "A10")
wb.save("C:\****\****\testFile.xlsx")```

Let's say you want your linear trend line to be green. Then...
from openpyxl.chart.trendline import Trendline
from openpyxl.chart.shapes import GraphicalProperties
from openpyxl.drawing.line import LineProperties
line_props = LineProperties(solidFill='00FF00')
g_props = GraphicalProperties(ln=line_props)
linear_trendline = Trendline(spPr=g_props)
my_chart.series[0].trendline = linear_trendline

ValueError: could not convert string to float: '1.csv'

I get a ValueError: could not convert string to float: '1.csv' on Python 3.6.3 with the following code:
import numpy as np
import matplotlib.pyplot as plt
from lmfit import Model
'''####### functions #####'''
def moving_average(a, n=3) :
ret = np.cumsum(a, dtype=float)
ret[n:] = ret[n:] - ret[:-n]
return ret[n - 1:] / n
# Fitting function
def gaussian(x,a,x0,w):
return a*np.exp(-2*(x-x0)**2/w**2)
'''####### file name ######'''
savfile = False
file_n = ['1','2','3','4','5','6','7','8','9','10',
'11','12','13','14','15','16','17','18','19','20','21','22','23','24']
# hole number of reference
dist = [3,5,7,9,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,27,29,32,36,38]
dist = np.array(dist,dtype=float)*1e-3*25 +1e-3*25 +1e-3*230
'''####### flags representing which point on chopper #######'''
'''####### the razor initiates the plot, as beam diverges #######'''
flag = [0,0,1,1,0,0,1,0,0,0,1,1,1,0,1,0,1,1,0,0,0,0,0,0]
'''####### empty arrays of zero ready to be filled #######'''
w_max_mean = np.zeros((2,np.size(dist)))
w_min_mean = np.zeros((2,np.size(dist)))
w_max_stderr = np.zeros((2,np.size(dist)))
w_min_stderr = np.zeros((2,np.size(dist)))
for k in range(np.size(file_n)):#range(1):#
file_name = file_n[k] + ".csv"
file_name1 = float(file_name)
#file_name = float(file_name)
# print(file_name)
data = np.loadtxt(file_name1,comments='%',delimiter=',',usecols=[0,1],skiprows=0)
time0 = data[:,0]
volt0 = data[:,1]
n_int = 10.#+ np.abs(k-15)
t = np.zeros(np.array(np.size(t0)/n_int,dtype = int)-1);
V = np.zeros(np.array(np.size(t0)/n_int,dtype = int)-1);
for j in range(np.array(np.size(t0)/n_int,dtype = int)-1):
t[j] = np.sum(t0[j*n_int:(j+1)*n_int-1])/n_int
V[j] = np.sum(V0[j*n_int:(j+1)*n_int-1])/n_int
What causes this?

[np.loadtxt]1 takes a str as it's first argument, not a float. Why are you trying to convert the filename to a float? Just remove file_name1 = float(file_name) and replace
data = np.loadtxt(file_name1,comments='%',delimiter=',',usecols=[0,1],skiprows=0)
with
data = np.loadtxt(file_name, comments='%', delimiter=',', usecols=[0,1], skiprows=0)
and it should all work.

Add each new dictionary result in the order of the columns of a dataframe

I am new to Python, but hope to explain the issue.
dfrow - is a dictionary of a single regression summary
results - is an empty dataframe with same columns as in dfrow
I would like to save regression results for each observation in the outer loop at the same time making sure column order in the inner loop. I am getting a result for the first observations but cannot move further, error saying:
Traceback (most recent call last):
File "<stdin>", line 109, in <module>
TypeError: 'numpy.int64' object is not iterable
when I run this code
import pandas as pd
import numpy as np
from statsmodels.tsa.arima_model import ARIMA
from statsmodels.stats import stattools as st
import statsmodels.api as sm
import collections
import datetime
import warnings
import scipy.stats
df_rent = import_rents()
df_return = import_ee_rets()
mostrecent = df_return.iloc[len(df_return) - 1]
mostrecentYYYY = mostrecent['Year']
mostrecentQ = mostrecent['Quarter']
mostrecentperiod = str(mostrecentYYYY) + "-Q" + str(mostrecentQ)
rentcols = df_rent.columns.values
colnames = []
#loop through the columns in df_rent until the column == the most recent period for which we have ee return data
for colname in rentcols:
if colname != mostrecentperiod:
colnames.append(colname)
else:
colnames.append(colname)
break
rentcols = colnames
#subset df_rent to only include columns that also have ee return data
df_rent = df_rent[rentcols]
#change dtype of metro_code / metro columns to string for matching later
df_rent['metro_code'] = df_rent['metro_code'].apply(str)
df_return['Metro'] = df_return['Metro'].apply(str)
df = pd.read_csv('//x/Project/_data/raw_data/rent_change.csv')
metros = list(np.unique(df['metro_code']))
regress_result_names = [
'metro',
'num_lag',
'num_ma',
'num_AR',
'beta_x1_retmov',
'x1_se',
'x1_tstat',
'x1_pval',
'r-squared',
'reg_fstat',
'fstat_pvalue',
'durbin-watson',
'resid_var']
regress_result_names = pd.Series(regress_result_names)
results = pd.DataFrame(columns=regress_result_names)
row = 0
for metro in metros:
for nlag in range(0, 5):
for nma in range(1, 11):
for AR in range(1, 5):
y = df_rent[df_rent['metro_code'] == str(metro)]
y = y.values.tolist()
y = y[0]
# delete first two columns of df_rent (they don't contain numeric data)
y.pop(0)
y.pop(0)
#y = rent time series data for specific metro
y = pd.Series(y)
#x1 = lagged moving average data for given params
df_return1 = df_return[df_return['Metro'] == str(metro)]
df_return1 = df_return1.reset_index(drop = True)
x1 = lagged_moving_avg(df = df_return1, metro_code = metro, nlag = nlag, nma = nma)
#y and x1 dataframe
y_label = 'y_Rent'
x_lagMA_label = 'x1_LaggedMA'
df1 = pd.DataFrame()
df1[y_label] = y
df1[x_lagMA_label] = x1
if mostrecentQ == 1:
currmonth = "01"
elif mostrecentQ == 2:
currmonth = "04"
elif mostrecentQ == 3:
currmonth = "07"
else:
currmonth = "10"
#convert index to datetime to run the regressions
currpd = pd.to_datetime((str(mostrecentYYYY) + currmonth), format='%Y%m')
df1.index = pd.date_range(*(pd.to_datetime(['1990-01', currpd]) + pd.offsets.QuarterEnd()), freq='Q')
#drop any rows that have missing observations
df1 = df1.dropna()
#df1.to_csv('//Nisfile01/x/Project - Real Estate Database/real_estate/odil/XandY.csv', index=True)
reg = ARIMA(endog = df1[y_label], order = (AR, 0,0)).fit(trend = 'nc', disp = 0, tol=1e-20)
resid_reg = reg.resid
reg2 = sm.OLS(resid_reg, df1[x_lagMA_label]).fit()
resid_reg2 = reg2.resid
dfrow = {
'metro': metro,
'num_lag': nlag,
'num_ma': nma,
'num_AR': AR,
'beta_x1_retmov': reg2.params[0],
'x1_se': reg2.bse[0],
'x1_tstat': reg2.tvalues[0],
'x1_pval': reg2.pvalues[0],
'r-squared': reg2.rsquared,
'reg_fstat':reg2.fvalue,
'fstat_pvalue': reg2.f_pvalue,
'durbin-watson': st.durbin_watson(reg2.resid),
'resid_var': resid_reg2.var(),
}
#create df for output called results
for key in dfrow.keys():
results.loc[row, key] = list(dfrow[key])
row = row + 1
Any help is very much appreciated.
P.S. Sorry for the messy code

The offending line is results.loc[row, key] = list(dfrow[key]).
You are trying to convert a single value, in this case a numpy.int64 object, to a list. I assume that what you're trying to do, and correct me if I am wrong, is create a singleton list with the int64 inside it. If that's what you want to do, you should use:
results.loc[row, key] = [dfrow[key]]

Looping same program for different data files

For the following program, I am trying to save time copying and pasting tons of code. I would like this program to plot using the data file 19_6.txt and aux.19_6, and then continue by plotting the files with 11,12,20,28,27, and 18 in 19's place with the same code and onto the same plot. Any help would be appreciated. Thanks!
from numpy import *
import matplotlib.pyplot as plt
datasim19 = loadtxt("/home/19_6.txt")
data19 = loadtxt("/home/aux.19_6")
no1=1
no2=2
no3=3
no4=4
no5=5
no7=7
no8=8
no9=9
no10=10
simrecno1inds19 = nonzero(datasim19[:,1]==no1)[0]
simrecno2inds19 = nonzero(datasim19[:,1]==no2)[0]
simrecno3inds19 = nonzero(datasim19[:,1]==no3)[0]
simrecno4inds19 = nonzero(datasim19[:,1]==no4)[0]
simrecno5inds19 = nonzero(datasim19[:,1]==no5)[0]
simrecno7inds19 = nonzero(datasim19[:,1]==no7)[0]
simrecno8inds19 = nonzero(datasim19[:,1]==no8)[0]
simrecno9inds19 = nonzero(datasim19[:,1]==no9)[0]
simrecno10inds19 = nonzero(datasim19[:,1]==no10)[0]
recno1inds19 = nonzero(data19[:,1]==no1)[0]
recno2inds19 = nonzero(data19[:,1]==no2)[0]
recno3inds19 = nonzero(data19[:,1]==no3)[0]
recno4inds19 = nonzero(data19[:,1]==no4)[0]
recno5inds19 = nonzero(data19[:,1]==no5)[0]
recno7inds19 = nonzero(data19[:,1]==no7)[0]
recno8inds19 = nonzero(data19[:,1]==no8)[0]
recno9inds19 = nonzero(data19[:,1]==no9)[0]
recno10inds19 = nonzero(data19[:,1]==no10)[0]
q1sim19 = qsim19[simrecno1inds19]
q2sim19 = qsim19[simrecno2inds19]
q3sim19 = qsim19[simrecno3inds19]
q4sim19 = qsim19[simrecno4inds19]
q5sim19 = qsim19[simrecno5inds19]
q7sim19 = qsim19[simrecno7inds19]
q8sim19 = qsim19[simrecno8inds19]
q9sim19 = qsim19[simrecno9inds19]
q10sim19 = qsim19[simrecno10inds19]
q1_19 = q19[recno1inds19]
q2_19 = q19[recno2inds19]
q3_19 = q19[recno3inds19]
q4_19 = q19[recno4inds19]
q5_19 = q19[recno5inds19]
q7_19 = q19[recno7inds19]
q8_19 = q19[recno8inds19]
q9_19 = q19[recno9inds19]
q10_19 = q19[recno10inds19]
sumq1sim19 = sum(q1sim19)
sumq2sim19 = sum(q2sim19)
sumq3sim19 = sum(q3sim19)
sumq4sim19 = sum(q4sim19)
sumq5sim19 = sum(q5sim19)
sumq7sim19 = sum(q7sim19)
sumq8sim19 = sum(q8sim19)
sumq9sim19 = sum(q9sim19)
sumq10sim19 = sum(q10sim19)
sumq1_19 = sum(q1_19)
sumq2_19 = sum(q2_19)
sumq3_19 = sum(q3_19)
sumq4_19 = sum(q4_19)
sumq5_19 = sum(q5_19)
sumq7_19 = sum(q7_19)
sumq8_19 = sum(q8_19)
sumq9_19 = sum(q9_19)
sumq10_19 = sum(q10_19)
xsim = [no1, no2, no3, no4, no5, no7, no8, no9, no10]
ysim = [sumq1sim_19, sumq2sim_19, sumq3sim_19, sumq4sim_19, sumq5sim_19, sumq7sim_19, sumq8sim_19, sumq9sim_19, sumq10sim_19]
x = [no1, no2, no3, no4, no5,no7, no8, no9, no10]
y = [sumq1_19, sumq2_19, sumq3_19, sumq4_19, sumq5_19, sumq7_19, sumq8_19, sumq9_19, sumq10_19]
plt.plot(x,log(y),'b',label='Data')
plt.plot(xsim,log(ysim),'r',label='Simulation')
plt.legend()
plt.title('Data vs. Simulation')
plt.show()

Tip: when you find yourself using lots of variables called n1, n2, n3 etc. you should probably use lists, dictionaries or other such containers, and loops instead.
For example, try replacing the following code:
simrecno1inds19 = nonzero(datasim19[:,1]==no1)[0]
simrecno2inds19 = nonzero(datasim19[:,1]==no2)[0]
simrecno3inds19 = nonzero(datasim19[:,1]==no3)[0]
simrecno4inds19 = nonzero(datasim19[:,1]==no4)[0]
simrecno5inds19 = nonzero(datasim19[:,1]==no5)[0]
simrecno7inds19 = nonzero(datasim19[:,1]==no7)[0]
simrecno8inds19 = nonzero(datasim19[:,1]==no8)[0]
simrecno9inds19 = nonzero(datasim19[:,1]==no9)[0]
simrecno10inds19 = nonzero(datasim19[:,1]==no10)[0]
With this:
simrecinds19 = [nonzero(datasim19[:,1] == i)[0] for i in range(1, 11)]
Then you can use simrecinds19[0] instead of simrecno1inds19.

You can do something like this:
nList = [19,11,12,20,28,27,18]
for n in nList:
file1 = "/home/" + str(n) + "_6.txt"
file2 = "/home/aux." + str(n) + "_6"
datasim19 = loadtxt(file1)
data19 = loadtxt(file2)
# do the rest of the plotting

You can greatly reduce the size of this script. I'm not quite sure where qsim19 and qsim come from, but take a look:
import numpy as np
import matplotlib.pyplot as plt
for index in [19, 11, 12, 20, 28, 27, 18]:
datasim = loadtxt("/home/%i_6.txt"%index)
data = loadtxt("/home/aux.%i_6"%index)
nos = range(1, 6) + range(7, 11)
simrecno = [np.nonzero(datasim[:,1] == n)[0] for n in nos]
recno = [np.nonzero(data[:,1] == n)[0] for n in nos]
qsim = [qsim[simrecno_i] for simrecno_i in simrecno]
q = [q[recno_i] for recno_i in recno]
sumqsim = [sum(qsim_i) for qsim_i in qsim]
sumq = [sum(q_i) for q_i in q]
xsim = nos
ysim = sumqsim
x = nos
y = sumq
plt.plot(x, log(y), 'b', label='Data')
plt.plot(xsim, log(ysim), 'r', label='Simulation')
plt.legend()
plt.title('Data vs. Simulation')
plt.show()