I want to fetch stock information from Moneycontrol site. I want to fill stock name in searchbox.
html code:
<input class="txtsrchbox FL" id="search_str"
onkeyup="getAutosuggesionHeader('#form_topsearch');"
onclick="getAutosuggesionHeader('#form_topsearch');"
placeholder="Search Quotes, News, NAVs" name="search_str"
value="" type="text" autocomplete="off">
How can i achieve this using beautifulsoup.
I have tried following code:
from BeautifulSoup import BeautifulSoup
import urllib
post_params = {'value' : 'SBIN' }
post_args = urllib.parse.urlencode(post_params).encode("utf-8")
url = 'https://www.moneycontrol.com'
fp = urllib.request.urlopen(url, post_args)
soup = BeautifulSoup(fp)
I'm not familiar with financial info so this might not be of use, but maybe it is, or maybe QHarr can work off of it. Looks like it gives back some value for (t:time?, cp:closing price? v:volume? ap:???)
So I noticed you can get some response from the chart data, however it requires to enter a parameter to query (ie. a sc_id value, in this case for SBIN, is SBI.)
So I needed a way though to get that sc_id value, and saw they have a "suggested" text search return that can be accessed to enter your search word (ie: "sbin"), to get back that sc_id.
So here's a little script that was able to get some info back for SBIN. Hopefully this can get some use out of it:
import requests
import json
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'}
# Enter search text
query_input = input('Enter Search Text: ')
#Get suggested sc_id
suggest_query_url = 'https://www.moneycontrol.com/mccode/common/autosuggestion_solr.php'
query = {
'classic': 'true',
'query': query_input,
'type': '1',
'format': 'json',
'callback': 'suggest1'}
# Pull out the sc_id
suggested_response = requests.get(suggest_query_url , headers=headers, params=query).text
suggested_response = suggested_response.split('(',1)[1]
suggested_response = suggested_response.rsplit(')',1)[0]
sc_id = json.loads(suggested_response)[0]['sc_id']
# Use the sc_id to get BSE and NSE data
request_url = 'https://www.moneycontrol.com/stocks/company_info/get_vwap_chart_data.php'
query = {'sc_did': sc_id}
response = requests.get(request_url, headers=headers, params=query).json()
Output:
print (response)
{'BSE': [{'t': '1551949665', 'ap': '279.70', 'cp': '279.70', 'v': '2151'}, {'t': '1551949704', 'ap': '279.70', 'cp': '279.70', 'v': '2151'}, {'t': '1551949740', 'ap': '279.70', 'cp': '279.70', 'v': '2151'}, {'t': '1551950159', 'ap': '278.93', 'cp': '278.90', 'v': '18755'}, {'t': '1551950219', 'ap': '278.90', 'cp': '278.70', 'v': '23368'}, {'t': '1551950279', 'ap': '278.89', 'cp': '279.00', 'v': '32498'}, {'t': '1551950338', 'ap': '278.91', 'cp': '279.00', 'v': '36396'}, {'t': '1551950399', 'ap': '278.90', 'cp': '278.80', 'v': '42964'}, {'t': '1551950459', 'ap': '278.88', 'cp': '278.35', 'v': '45685'}, {'t': '1551950519', 'ap': '278.76', 'cp': '278.30', 'v': '54082'}, {'t': '1551950579', 'ap': '278.74', 'cp': '278.30', 'v': '56780'}, {'t': '1551950639', 'ap': '278.69', 'cp': '278.20', 'v': '62504'}, {'t': '1551950699', 'ap': '278.68', 'cp': '278.10', 'v': '63338'}, {'t': '1551950759', 'ap': '278.68', 'cp': '278.10', 'v': '63723'}, {'t': '1551950819', 'ap': '278.67', 'cp': '277.80', 'v': '64998'}, {'t': '1551950879', 'ap': '278.63', 'cp': '278.20', 'v': '68780'}, {'t': '1551950939', 'ap': '278.59', 'cp': '278.60', 'v': '77680'}, {'t': '1551950999', 'ap': '278.59', 'cp': '278.35', 'v': '79316'}, {'t': '1551951059', 'ap': '278.58', 'cp': '278.30', 'v': '80566'}, {'t': '1551951119', 'ap': '278.58', 'cp': '278.20', 'v': '81247'}, {'t': '1551951178', 'ap': '278.57', 'cp': '278.10', 'v': '82067'}, {'t': '1551951238', 'ap': '278.57', 'cp': '278.15', 'v': '82918'}, {'t': '1551951294', 'ap': '278.56', 'cp': '278.60', 'v': '85734'}, {'t': '1551951358', 'ap': '278.56', 'cp': '278.40', 'v': '87239'}, {'t': '1551951419', 'ap': '278.56', 'cp': '278.25', 'v': '88039'}, ...
Related
I have a variable which stores below dictionary
initial_ltp =
{'s': 'ok',
'd': [{'n': 'MCX:CRUDEOIL23JANFUT',
's': 'ok',
'v': {'ch': 47.0,
'chp': 0.74,
'lp': 6377.0,
'spread': 2.0,
'ask': 6379.0,
'bid': 6377.0,
'open_price': 6330.0,
'high_price': 6393.0,
'low_price': 6305.0,
'prev_close_price': 6330.0,
'volume': 8410,
'short_name': 'CRUDEOIL23JANFUT',
'exchange': 'MCX',
'description': 'MCX:CRUDEOIL23JANFUT',
'original_name': 'MCX:CRUDEOIL23JANFUT',
'symbol': 'MCX:CRUDEOIL23JANFUT',
'fyToken': '1120230119244999',
'tt': 1673481600,
'cmd': {'t': 1673518200,
'o': 6376.0,
'h': 6378.0,
'l': 6375.0,
'c': 6377.0,
'v': 19,
'tf': '15:40'}}},
{'n': 'MCX:SILVERMIC23FEBFUT',
's': 'ok',
'v': {'ch': 485.0,
'chp': 0.71,
'lp': 68543.0,
'spread': 5.0,
'ask': 68545.0,
'bid': 68540.0,
'open_price': 68200.0,
'high_price': 68689.0,
'low_price': 68200.0,
'prev_close_price': 68058.0,
'volume': 49595,
'short_name': 'SILVERMIC23FEBFUT',
'exchange': 'MCX',
'description': 'MCX:SILVERMIC23FEBFUT',
'original_name': 'MCX:SILVERMIC23FEBFUT',
'symbol': 'MCX:SILVERMIC23FEBFUT',
'fyToken': '1120230228242738',
'tt': 1673481600,
'cmd': {'t': 1673518200,
'o': 68525.0,
'h': 68543.0,
'l': 68524.0,
'c': 68543.0,
'v': 140,
'tf': '15:40'}}}]}
I am trying to collect ('n') and ('lp') and save it in a different dictionary using code:
if 'd' in initial_ltp.keys():
ltp[initial_ltp['d'][0]['n']] = initial_ltp['d'][0]['v']['lp']
But it is only taking the first n and lp
ltp
{'MCX:CRUDEOIL23JANFUT': 6377.0}
My expected output:
ltp
{'MCX:CRUDEOIL23JANFUT': 6377.0, 'MCX:SILVERMIC23FEBFUT': 68543.0}
How can I get both the values
You have to loop over the list. Using ltp[initial_ltp['d'][0] will just extract for the first element of the list.
Here is an example:
results = {}
for doc in initial_ltp["d"]:
results[doc["n"]] = doc["v"]["lp"]
print(results)
Use the following approach with dict comprehension:
ltp = {d['n']:d['v']['lp'] for d in initial_ltp['d']} if 'd' in initial_ltp else {}
{'MCX:CRUDEOIL23JANFUT': 6377.0, 'MCX:SILVERMIC23FEBFUT': 68543.0}
when you use the "=" operator, it replaces your value in the dictionary key.
you want to add keys to your dictionary so I suggest using this:
if 'd' in initial_ltp.keys():
for o in initial_ltp['d']:
if n in o:
ltp[initial_ltp['d'][0]['n']].append(initial_ltp['d'][0]['v']
['lp']
It's because you selected only first item in the 'd'.
try a loop like this:
ltp={}
if 'd' in initial_ltp.keys():
for i in range(len(initial_ltp['d'])):
ltp[initial_ltp['d'][i]['n']] = initial_ltp['d'][i]['v']['lp']
print (ltp)
Output:
{'MCX:CRUDEOIL23JANFUT': 6377.0, 'MCX:SILVERMIC23FEBFUT': 68543.0}
Am getting API response Like Below
https://imgur.com/a/uVcLfF4
{'s': 'ok', 'd':
[
{'n': 'NSE:SBIN22JUL485CE', 's': 'ok', 'v': {'ch': -2.25, 'chp': -17.05, 'lp': 10.95, 'spread': 0.25, 'ask': 10.95, 'bid': 10.7, 'open_price': 11.5, 'high_price': 15.05, 'low_price': 10.45, 'prev_close_price': 13.2, 'volume': 1161000, 'short_name': 'SBIN22JUL485CE', 'exchange': 'NSE', 'description': 'NSE:SBIN22JUL485CE', 'original_name': 'NSE:SBIN22JUL485CE', 'symbol': 'NSE:SBIN22JUL485CE', 'fyToken': '1011220728149794', 'tt': 1657584000, 'cmd': {'t': 1657620000, 'o': 10.95, 'h': 10.95, 'l': 10.95, 'c': 10.95, 'v': 1500, 'tf': '15:30'}}
},
{'n': 'NSE:SBIN22JUL480CE', 's': 'ok', 'v': {'ch': -2.65, 'chp': -16.46, 'lp': 13.45, 'spread': 0.1, 'ask':
13.45, 'bid': 13.35, 'open_price': 15.3, 'high_price': 18.45, 'low_price': 12.9, 'prev_close_price': 16.1, 'volume': 4270500, 'short_name': 'SBIN22JUL480CE', 'exchange': 'NSE', 'description': 'NSE:SBIN22JUL480CE', 'original_name': 'NSE:SBIN22JUL480CE', 'symbol': 'NSE:SBIN22JUL480CE', 'fyToken': '1011220728128799', 'tt': 1657584000, 'cmd': {'t': 1657619940, 'o': 13.45, 'h': 13.45, 'l': 13.45, 'c': 13.45, 'v': 28500, 'tf': '15:29'}}
}
]
}
How to read and print this in python like below.
Name= NSE:SBIN22JUL485CE
ch = -2.25
chp = -17.05
volume = 1161000
Name= NSE:SBIN22JUL480CE
ch = -2.65
chp = -16.46
volume = 4270500
It looks like json, if you have that text in a variable, you can use the python json librarie for decoding it, and get a dictionary.
import json
text = '...' # The variable that contains the response
data = json.loads(text)
for entry in data['d']:
print(f'Name = {entry["n"]}')
print(f'ch = {entry["v"]["ch"]}')
print(f'chp = {entry["v"]["chp"]}')
print(f'volume = {entry["v"]["volume"]}')
print('')
If for some reason, in the text, the quotes are single quotes ' insted of double ", you should need to replace it, before the json parsing:
text = text.replace("'", '"')
If it is already a dict, just print the content:
mydict = <API response>
if 's' in mydict and mydict['s'] == 'ok':
for data in mydict['d']:
print('Name:', data['n'])
print('ch:', data['v']['ch'])
print('chp:', data['v']['chp'])
print('volume:', data['v']['volume'])
print()
If it is not a dict then you need to convert the content before:
import json
content = <API response>
mydict = json.loads(content)
<code to print above>
I am using a third party python library, which returns some data in lists and dictionaries inside them (JSON format). For example, here's a sample entry:
data = [{'a': 1, 'b': 80, 'c': 42, 'd': [{'r': 0, 's': '1', 'u': 5}], 'id': 10, 'k': 60, 'v': 0, 'm':
{'ty': 'djp', 'nr': '10', 'bc': Adder('179'), 'in': 3}}, {'a': 1, 'b': 70, 'c': 42, 'd': [{'r': 0, 's':
'1', 'u': 5}], 'y': 10, 'k': 60, 'v': 0, 'm': {'ty': 'djp', 'dx': '10', 'tx': Adder('179'), 'in': 3}}]
My problem is with 'Adder' class which is an object. Everything else are just strings.
So, when I do:
json.dumps(data)
It causes an error message:
Adder('179') is not JSON serializable.
I don't care about serializing this Adder class, so I would somehow like to just make it a string e.g. "Adder(179)" if possible. Is it possible to make the dumps function work? Class Adder is not my own, maybe I can make it serialized or tell it what to do with it by editing the source or something? Any simple way would be fine.
Just specify a custom encoder class:
class RobustEncoder(json.JSONEncoder):
def default(self, o):
try:
return super(RobustEncoder, self).default(o)
except TypeError:
return str(o)
json.dumps(data, cls=RobustEncoder)
The keyword argument cls is documented in the docs of json.dump.
I have a function which results
dict1 = {'2132': [{'L': {'Y': '452.2'}}, {'L': {'N': '21'}}], '2345': [{'L': {'Y': '87'}}, {'C': {'N': '56'}}, {'6': {'Y': '45.23'}}]
I have another function i need to pass 2132,L,Y values from dict1 as an arguments and it should result 452.2
def getx(a, b, c):
try:
return dict1[a][b][c]
except:
return None
when i give dict1['2132'] which results [{'L': {'Y': '452.2'}}, {'L': {'N': '21'}}]
i want dict1['2132']['L']['Y'] should result as 452.2
so i need my dictionary
dict1 = {'2132': [{'L': {'Y': '452.2'}}, {'L': {'N': '21'}}], '2345': [{'L': {'Y': '87'}}, {'C': {'N': '56'}}, {'6': {'Y': '45.23'}}]
to show as
dict1 = {'2132': {{'L': {'Y': '452.2'}}, {'L': {'N': '21'}}}, '2345': {{'L': {'Y': '87'}}, {'C': {'N': '56'}}, {'6': {'Y': '45.23'}}}
OR is there any other way that we can pull the 4th value when the dict1 is
dict1 = {'2132': [{'L': {'Y': '452.2'}}, {'L': {'N': '21'}}], '2345': [{'L': {'Y': '87'}}, {'C': {'N': '56'}}, {'6': {'Y': '45.23'}}]
here is your solution
#v1='2132' v2='L' v3='Y'
def Solution(v1,v2,v3):
if v1 in dict1.keys():
for i in dict1[v1]:
if v2 in i.keys():
if v3 in i[v2]:
return i[v2][v3]
return None
dict1 = {'2132': [{'L': {'Y': '452.2'}}, {'C': {'N': '21'}}], '2345': [{'L': {'Y': '87'}}, {'C': {'N': '56'}},{'6': {'Y': '45.23'}}]}
print(Solution('2132','L','Y'))
How about this:
from collections import defaultdict
for key,values in dict1.items():
temp_dict = defaultdict(dict)
for val in values: #values is a list of dict
for k,v in val.items():
temp_dict[k].update(v)
dict1[key] = dict(temp_dict)
print(dict1)
#{'2132': {'L': {'Y': '452.2', 'N': '21'}}, '2345': {'L': {'Y': '87'}, 'C': {'N': '56'}, '6': {'Y': '45.23'}}}
And then
def getx(a, b, c):
try:
return dict1[a][b][c]
except:
return None
print(getx('2132','L','Y'))
#452.2
I'm trying to manipulate nested dictionaries to combine backwards the keys of any nested dictionaries with only a single key.
I've tried to do this recursively, but I'm having a hard time being able to remove keys from the dictionary and replace them with the concatenated keys.
For example:
{'adv':
{'e':
{'r':
{
's':
{'e':
{'_end_': '_end_'}
},
't':
{'_end_': '_end_',
'i':
{'s':
{'e':
{'r':
{'_end_': '_end_'}
}
}
}
}
}
},
'i': {'c': {'e': {'_end_': '_end_'}
}
}
}
}
would become
{'adv':
{'er':
{
'se':
{'_end_': '_end_'},
't':
{'_end_': '_end_',
'iser':
{'_end_': '_end_'}
}
},
'ice':
{'_end_': '_end_'}
}
}
This was an interesting problem - there is probably a more elegant solution, but I did the following:
import pprint
t={'adv': {'e': {'r': {'s': {'e': {'_end_': '_end_'}},
't': {'_end_': '_end_',
'i': {'s': {'e': {'r': {'_end_': '_end_'}}}}}}},
'i': {'c': {'e': {'_end_': '_end_'}}}}}
def concat_dict(d):
if d == '_end_':
return '_end_'
rv = {}
for k, v in d.items():
if '_end_' in v:
rv[k] = concat_dict(v)
elif len(list(x for x in v.keys() if x != '_end_')) == 1:
top_str = k
next_str = list(v.keys())[0]
rv[top_str + next_str] = concat_dict(v[next_str])
else:
rv[k] = concat_dict(v)
return rv
def format_dict(d):
while concat_dict(d) != d:
d = concat_dict(d)
return d
pprint.pprint(format_dict(t))
Output:
{'adv': {'er': {'se': {'_end_': '_end_'},
't': {'_end_': '_end_', 'iser': {'_end_': '_end_'}}},
'ice': {'_end_': '_end_'}}}