I have seen other posts about this error, and I have applied all the recommended so-called 'fixes', but none of them are working in my case.
I have windows 11, AMD 5700g, Radeon Rx 560, 32gb RAM.
So here's the error text in full:
Traceback (most recent call last):
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\__main__.py", line 181, in fmt_streams
extract.apply_signature(stream_manifest, self.vid_info, self.js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\extract.py", line 409, in apply_signature
cipher = Cipher(js=js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 43, in __init__
self.throttling_plan = get_throttling_plan(js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 405, in get_throttling_plan
raw_code = get_throttling_function_code(js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 311, in get_throttling_function_code
name = re.escape(get_throttling_function_name(js))
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 296, in get_throttling_function_name
raise RegexMatchError(
pytube.exceptions.RegexMatchError: get_throttling_function_name: could not find match for multiple
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Redje\Desktop\PythonFiles\YouTubeDownloader\YTD1.py", line 5, in <module>
stream = video.streams.get_highest_resolution()
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\__main__.py", line 296, in streams
return StreamQuery(self.fmt_streams)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\__main__.py", line 188, in fmt_streams
extract.apply_signature(stream_manifest, self.vid_info, self.js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\extract.py", line 409, in apply_signature
cipher = Cipher(js=js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 43, in __init__
self.throttling_plan = get_throttling_plan(js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 405, in get_throttling_plan
raw_code = get_throttling_function_code(js)
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 311, in get_throttling_function_code
name = re.escape(get_throttling_function_name(js))
File "C:\Users\Redje\AppData\Local\Programs\Python\Python310\lib\site-packages\pytube\cipher.py", line 296, in get_throttling_function_name
raise RegexMatchError(
pytube.exceptions.RegexMatchError: get_throttling_function_name: could not find match for multiple
I have tweaked 'cipher.py' according to an earlier post, and no luck. Any help would be appreciated.
Here is my cipher.py code:
"""
This module contains all logic necessary to decipher the signature.
YouTube's strategy to restrict downloading videos is to send a ciphered version
of the signature to the client, along with the decryption algorithm obfuscated
in JavaScript. For the clients to play the videos, JavaScript must take the
ciphered version, cycle it through a series of "transform functions," and then
signs the media URL with the output.
This module is responsible for (1) finding and extracting those "transform
functions" (2) maps them to Python equivalents and (3) taking the ciphered
signature and decoding it.
"""
import logging
import re
from itertools import chain
from typing import Any, Callable, Dict, List, Optional, Tuple
from pytube.exceptions import ExtractError, RegexMatchError
from pytube.helpers import cache, regex_search
from pytube.parser import find_object_from_startpoint, throttling_array_split
logger = logging.getLogger(__name__)
class Cipher:
def __init__(self, js: str):
self.transform_plan: List[str] = get_transform_plan(js)
var_regex = re.compile(r"^\w+\W")
var_match = var_regex.search(self.transform_plan[0])
if not var_match:
raise RegexMatchError(
caller="__init__", pattern=var_regex.pattern
)
var = var_match.group(0)[:-1]
self.transform_map = get_transform_map(js, var)
self.js_func_patterns = [
r"\w+\.(\w+)\(\w,(\d+)\)",
r"\w+\[(\"\w+\")\]\(\w,(\d+)\)"
]
self.throttling_plan = get_throttling_plan(js)
self.throttling_array = get_throttling_function_array(js)
self.calculated_n = None
def calculate_n(self, initial_n: list):
"""Converts n to the correct value to prevent throttling."""
if self.calculated_n:
return self.calculated_n
# First, update all instances of 'b' with the list(initial_n)
for i in range(len(self.throttling_array)):
if self.throttling_array[i] == 'b':
self.throttling_array[i] = initial_n
for step in self.throttling_plan:
curr_func = self.throttling_array[int(step[0])]
if not callable(curr_func):
logger.debug(f'{curr_func} is not callable.')
logger.debug(f'Throttling array:\n{self.throttling_array}\n')
raise ExtractError(f'{curr_func} is not callable.')
first_arg = self.throttling_array[int(step[1])]
if len(step) == 2:
curr_func(first_arg)
elif len(step) == 3:
second_arg = self.throttling_array[int(step[2])]
curr_func(first_arg, second_arg)
self.calculated_n = ''.join(initial_n)
return self.calculated_n
def get_signature(self, ciphered_signature: str) -> str:
"""Decipher the signature.
Taking the ciphered signature, applies the transform functions.
:param str ciphered_signature:
The ciphered signature sent in the ``player_config``.
:rtype: str
:returns:
Decrypted signature required to download the media content.
"""
signature = list(ciphered_signature)
for js_func in self.transform_plan:
name, argument = self.parse_function(js_func) # type: ignore
signature = self.transform_map[name](signature, argument)
logger.debug(
"applied transform function\n"
"output: %s\n"
"js_function: %s\n"
"argument: %d\n"
"function: %s",
"".join(signature),
name,
argument,
self.transform_map[name],
)
return "".join(signature)
#cache
def parse_function(self, js_func: str) -> Tuple[str, int]:
"""Parse the Javascript transform function.
Break a JavaScript transform function down into a two element ``tuple``
containing the function name and some integer-based argument.
:param str js_func:
The JavaScript version of the transform function.
:rtype: tuple
:returns:
two element tuple containing the function name and an argument.
**Example**:
parse_function('DE.AJ(a,15)')
('AJ', 15)
"""
logger.debug("parsing transform function")
for pattern in self.js_func_patterns:
regex = re.compile(pattern)
parse_match = regex.search(js_func)
if parse_match:
fn_name, fn_arg = parse_match.groups()
return fn_name, int(fn_arg)
raise RegexMatchError(
caller="parse_function", pattern="js_func_patterns"
)
def get_initial_function_name(js: str) -> str:
"""Extract the name of the function responsible for computing the signature.
:param str js:
The contents of the base.js asset file.
:rtype: str
:returns:
Function name from regex match
"""
function_patterns = [
r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # noqa: E501
r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # noqa: E501
r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
r"\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(",
r"yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(", # noqa: E501
]
logger.debug("finding initial function name")
for pattern in function_patterns:
regex = re.compile(pattern)
function_match = regex.search(js)
if function_match:
logger.debug("finished regex search, matched: %s", pattern)
return function_match.group(1)
raise RegexMatchError(
caller="get_initial_function_name", pattern="multiple"
)
def get_transform_plan(js: str) -> List[str]:
"""Extract the "transform plan".
The "transform plan" is the functions that the ciphered signature is
cycled through to obtain the actual signature.
:param str js:
The contents of the base.js asset file.
**Example**:
['DE.AJ(a,15)',
'DE.VR(a,3)',
'DE.AJ(a,51)',
'DE.VR(a,3)',
'DE.kT(a,51)',
'DE.kT(a,8)',
'DE.VR(a,3)',
'DE.kT(a,21)']
"""
name = re.escape(get_initial_function_name(js))
pattern = r"%s=function\(\w\){[a-z=\.\(\"\)]*;(.*);(?:.+)}" % name
logger.debug("getting transform plan")
return regex_search(pattern, js, group=1).split(";")
def get_transform_object(js: str, var: str) -> List[str]:
"""Extract the "transform object".
The "transform object" contains the function definitions referenced in the
"transform plan". The ``var`` argument is the obfuscated variable name
which contains these functions, for example, given the function call
``DE.AJ(a,15)`` returned by the transform plan, "DE" would be the var.
:param str js:
The contents of the base.js asset file.
:param str var:
The obfuscated variable name that stores an object with all functions
that descrambles the signature.
**Example**:
>>> get_transform_object(js, 'DE')
['AJ:function(a){a.reverse()}',
'VR:function(a,b){a.splice(0,b)}',
'kT:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c}']
"""
pattern = r"var %s={(.*?)};" % re.escape(var)
logger.debug("getting transform object")
regex = re.compile(pattern, flags=re.DOTALL)
transform_match = regex.search(js)
if not transform_match:
raise RegexMatchError(caller="get_transform_object", pattern=pattern)
return transform_match.group(1).replace("\n", " ").split(", ")
def get_transform_map(js: str, var: str) -> Dict:
"""Build a transform function lookup.
Build a lookup table of obfuscated JavaScript function names to the
Python equivalents.
:param str js:
The contents of the base.js asset file.
:param str var:
The obfuscated variable name that stores an object with all functions
that descrambles the signature.
"""
transform_object = get_transform_object(js, var)
mapper = {}
for obj in transform_object:
# AJ:function(a){a.reverse()} => AJ, function(a){a.reverse()}
name, function = obj.split(":", 1)
fn = map_functions(function)
mapper[name] = fn
return mapper
def get_throttling_function_name(js: str) -> str:
"""Extract the name of the function that computes the throttling parameter.
:param str js:
The contents of the base.js asset file.
:rtype: str
:returns:
The name of the function used to compute the throttling parameter.
"""
function_patterns = [
# https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-865985377
# https://github.com/yt-dlp/yt-dlp/commit/48416bc4a8f1d5ff07d5977659cb8ece7640dcd8
# var Bpa = [iha];
# ...
# a.C && (b = a.get("n")) && (b = Bpa[0](b), a.set("n", b),
# Bpa.length || iha("")) }};
# In the above case, `iha` is the relevant function name
r'a\.[a-zA-Z]\s*&&\s*\([a-z]\s*=\s*a\.get\("n"\)\)\s*&&\s*'
r'\([a-z]\s*=\s*([a-zA-Z0-9$]{2, 3})(\[\d+\])?\([a-z]\)',
]
logger.debug('Finding throttling function name')
for pattern in function_patterns:
regex = re.compile(pattern)
function_match = regex.search(js)
if function_match:
logger.debug("finished regex search, matched: %s", pattern)
if len(function_match.groups()) == 1:
return function_match.group(1)
idx = function_match.group(2)
if idx:
idx = idx.strip("[]")
array = re.search(
r'var {nfunc}\s*=\s*(\[.+?\]);'.format(
nfunc=escape(function_match.group(1))),
js
)
if array:
array = array.group(1).strip("[]").split(",")
array = [x.strip() for x in array]
return array[int(idx)]
raise RegexMatchError(
caller="get_throttling_function_name", pattern="multiple"
)
def get_throttling_function_code(js: str) -> str:
"""Extract the raw code for the throttling function.
:param str js:
The contents of the base.js asset file.
:rtype: str
:returns:
The name of the function used to compute the throttling parameter.
"""
# Begin by extracting the correct function name
name = re.escape(get_throttling_function_name(js))
# Identify where the function is defined
pattern_start = r"%s=function\(\w\)" % name
regex = re.compile(pattern_start)
match = regex.search(js)
# Extract the code within curly braces for the function itself, and merge any split lines
code_lines_list = find_object_from_startpoint(js, match.span()[1]).split('\n')
joined_lines = "".join(code_lines_list)
# Prepend function definition (e.g. `Dea=function(a)`)
return match.group(0) + joined_lines
def get_throttling_function_array(js: str) -> List[Any]:
"""Extract the "c" array.
:param str js:
The contents of the base.js asset file.
:returns:
The array of various integers, arrays, and functions.
"""
raw_code = get_throttling_function_code(js)
array_start = r",c=\["
array_regex = re.compile(array_start)
match = array_regex.search(raw_code)
array_raw = find_object_from_startpoint(raw_code, match.span()[1] - 1)
str_array = throttling_array_split(array_raw)
converted_array = []
for el in str_array:
try:
converted_array.append(int(el))
continue
except ValueError:
# Not an integer value.
pass
if el == 'null':
converted_array.append(None)
continue
if el.startswith('"') and el.endswith('"'):
# Convert e.g. '"abcdef"' to string without quotation marks, 'abcdef'
converted_array.append(el[1:-1])
continue
if el.startswith('function'):
mapper = (
(r"{for\(\w=\(\w%\w\.length\+\w\.length\)%\w\.length;\w--;\)\w\.unshift\(\w.pop\(\)\)}", throttling_unshift), # noqa:E501
(r"{\w\.reverse\(\)}", throttling_reverse),
(r"{\w\.push\(\w\)}", throttling_push),
(r";var\s\w=\w\[0\];\w\[0\]=\w\[\w\];\w\[\w\]=\w}", throttling_swap),
(r"case\s\d+", throttling_cipher_function),
(r"\w\.splice\(0,1,\w\.splice\(\w,1,\w\[0\]\)\[0\]\)", throttling_nested_splice), # noqa:E501
(r";\w\.splice\(\w,1\)}", js_splice),
(r"\w\.splice\(-\w\)\.reverse\(\)\.forEach\(function\(\w\){\w\.unshift\(\w\)}\)", throttling_prepend), # noqa:E501
(r"for\(var \w=\w\.length;\w;\)\w\.push\(\w\.splice\(--\w,1\)\[0\]\)}", throttling_reverse), # noqa:E501
)
found = False
for pattern, fn in mapper:
if re.search(pattern, el):
converted_array.append(fn)
found = True
if found:
continue
converted_array.append(el)
# Replace null elements with array itself
for i in range(len(converted_array)):
if converted_array[i] is None:
converted_array[i] = converted_array
return converted_array
def get_throttling_plan(js: str):
"""Extract the "throttling plan".
The "throttling plan" is a list of tuples used for calling functions
in the c array. The first element of the tuple is the index of the
function to call, and any remaining elements of the tuple are arguments
to pass to that function.
:param str js:
The contents of the base.js asset file.
:returns:
The full function code for computing the throttlign parameter.
"""
raw_code = get_throttling_function_code(js)
transform_start = r"try{"
plan_regex = re.compile(transform_start)
match = plan_regex.search(raw_code)
transform_plan_raw = find_object_from_startpoint(raw_code, match.span()[1] - 1)
# Steps are either c[x](c[y]) or c[x](c[y],c[z])
step_start = r"c\[(\d+)\]\(c\[(\d+)\](,c(\[(\d+)\]))?\)"
step_regex = re.compile(step_start)
matches = step_regex.findall(transform_plan_raw)
transform_steps = []
for match in matches:
if match[4] != '':
transform_steps.append((match[0],match[1],match[4]))
else:
transform_steps.append((match[0],match[1]))
return transform_steps
def reverse(arr: List, _: Optional[Any]):
"""Reverse elements in a list.
This function is equivalent to:
.. code-block:: javascript
function(a, b) { a.reverse() }
This method takes an unused ``b`` variable as their transform functions
universally sent two arguments.
**Example**:
>>> reverse([1, 2, 3, 4])
[4, 3, 2, 1]
"""
return arr[::-1]
def splice(arr: List, b: int):
"""Add/remove items to/from a list.
This function is equivalent to:
.. code-block:: javascript
function(a, b) { a.splice(0, b) }
**Example**:
>>> splice([1, 2, 3, 4], 2)
[1, 2]
"""
return arr[b:]
def swap(arr: List, b: int):
"""Swap positions at b modulus the list length.
This function is equivalent to:
.. code-block:: javascript
function(a, b) { var c=a[0];a[0]=a[b%a.length];a[b]=c }
**Example**:
>>> swap([1, 2, 3, 4], 2)
[3, 2, 1, 4]
"""
r = b % len(arr)
return list(chain([arr[r]], arr[1:r], [arr[0]], arr[r + 1 :]))
def throttling_reverse(arr: list):
"""Reverses the input list.
Needs to do an in-place reversal so that the passed list gets changed.
To accomplish this, we create a reversed copy, and then change each
indvidual element.
"""
reverse_copy = arr.copy()[::-1]
for i in range(len(reverse_copy)):
arr[i] = reverse_copy[i]
def throttling_push(d: list, e: Any):
"""Pushes an element onto a list."""
d.append(e)
def throttling_mod_func(d: list, e: int):
"""Perform the modular function from the throttling array functions.
In the javascript, the modular operation is as follows:
e = (e % d.length + d.length) % d.length
We simply translate this to python here.
"""
return (e % len(d) + len(d)) % len(d)
def throttling_unshift(d: list, e: int):
"""Rotates the elements of the list to the right.
In the javascript, the operation is as follows:
for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())
"""
e = throttling_mod_func(d, e)
new_arr = d[-e:] + d[:-e]
d.clear()
for el in new_arr:
d.append(el)
def throttling_cipher_function(d: list, e: str):
"""This ciphers d with e to generate a new list.
In the javascript, the operation is as follows:
var h = [A-Za-z0-9-_], f = 96; // simplified from switch-case loop
d.forEach(
function(l,m,n){
this.push(
n[m]=h[
(h.indexOf(l)-h.indexOf(this[m])+m-32+f--)%h.length
]
)
},
e.split("")
)
"""
h = list('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_')
f = 96
# by naming it "this" we can more closely reflect the js
this = list(e)
# This is so we don't run into weirdness with enumerate while
# we change the input list
copied_list = d.copy()
for m, l in enumerate(copied_list):
bracket_val = (h.index(l) - h.index(this[m]) + m - 32 + f) % len(h)
this.append(
h[bracket_val]
)
d[m] = h[bracket_val]
f -= 1
def throttling_nested_splice(d: list, e: int):
"""Nested splice function in throttling js.
In the javascript, the operation is as follows:
function(d,e){
e=(e%d.length+d.length)%d.length;
d.splice(
0,
1,
d.splice(
e,
1,
d[0]
)[0]
)
}
While testing, all this seemed to do is swap element 0 and e,
but the actual process is preserved in case there was an edge
case that was not considered.
"""
e = throttling_mod_func(d, e)
inner_splice = js_splice(
d,
e,
1,
d[0]
)
js_splice(
d,
0,
1,
inner_splice[0]
)
def throttling_prepend(d: list, e: int):
"""
In the javascript, the operation is as follows:
function(d,e){
e=(e%d.length+d.length)%d.length;
d.splice(-e).reverse().forEach(
function(f){
d.unshift(f)
}
)
}
Effectively, this moves the last e elements of d to the beginning.
"""
start_len = len(d)
# First, calculate e
e = throttling_mod_func(d, e)
# Then do the prepending
new_arr = d[-e:] + d[:-e]
# And update the input list
d.clear()
for el in new_arr:
d.append(el)
end_len = len(d)
assert start_len == end_len
def throttling_swap(d: list, e: int):
"""Swap positions of the 0'th and e'th elements in-place."""
e = throttling_mod_func(d, e)
f = d[0]
d[0] = d[e]
d[e] = f
def js_splice(arr: list, start: int, delete_count=None, *items):
"""Implementation of javascript's splice function.
:param list arr:
Array to splice
:param int start:
Index at which to start changing the array
:param int delete_count:
Number of elements to delete from the array
:param *items:
Items to add to the array
Reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice # noqa:E501
"""
# Special conditions for start value
try:
if start > len(arr):
start = len(arr)
# If start is negative, count backwards from end
if start < 0:
start = len(arr) - start
except TypeError:
# Non-integer start values are treated as 0 in js
start = 0
# Special condition when delete_count is greater than remaining elements
if not delete_count or delete_count >= len(arr) - start:
delete_count = len(arr) - start # noqa: N806
deleted_elements = arr[start:start + delete_count]
# Splice appropriately.
new_arr = arr[:start] + list(items) + arr[start + delete_count:]
# Replace contents of input array
arr.clear()
for el in new_arr:
arr.append(el)
return deleted_elements
def map_functions(js_func: str) -> Callable:
"""For a given JavaScript transform function, return the Python equivalent.
:param str js_func:
The JavaScript version of the transform function.
"""
mapper = (
# function(a){a.reverse()}
(r"{\w\.reverse\(\)}", reverse),
# function(a,b){a.splice(0,b)}
(r"{\w\.splice\(0,\w\)}", splice),
# function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c}
(r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\]=\w}", swap),
# function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}
(
r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\%\w.length\]=\w}",
swap,
),
)
for pattern, fn in mapper:
if re.search(pattern, js_func):
return fn
raise RegexMatchError(caller="map_functions", pattern="multiple")
Any help would be appreciated!
How would I get the value of a variable stored in a JSON file under a specific base / category using the existing solution below?
This is a class that takes JSON and stores it in a collection originally from
Multipurpose function to create and append json using specified base python
import json
import os
class data_manager():
BASE_COLLECTIONS_FOLDER = "./data"
def __init__(self, collection_name):
self.collection_name = collection_name
self.collection_file_path = (
f"{self.BASE_COLLECTIONS_FOLDER}/{self.collection_name}.json")
self.collection = {}
self.ensure_collection()
self.load_collection()
def ensure_collection(self):
if os.path.isfile(self.collection_file_path):
return
os.makedirs(self.BASE_COLLECTIONS_FOLDER, exist_ok=True)
self.save_collection()
def load_collection(self):
with open(self.collection_file_path, "r", encoding="utf-8") as collection_file:
self.collection = json.load(collection_file)[self.collection_name]
def save_collection(self):
with open(self.collection_file_path, "w", encoding="utf-8") as collection_file:
json.dump({self.collection_name: self.collection}, collection_file, indent=4)
def write_to_json(self, data, key=None):
if not key:
self.collection = {**self.collection, **data}
else:
self.collection[key] = {**self.collection.get(key, {}), **data}
self.save_collection()
people = data_manager("twitch")
streamer_information = {'streamer_username':None, 'streamer_username': None,
'streamer_game': None, 'streamer_uptime': None,
'streamer_viewers': None}
self.data.write_to_json({self.streamer_username: {}})
self.data.write_to_json(
{"online": self.online,
"streamer_title": self.streamer_information['streamer_title'],
"streamer_game": self.streamer_information['streamer_game'],
"streamer_uptime": self.streamer_information['streamer_uptime'],
"streamer_viewers": self.streamer_information['streamer_viewers']},
self.streamer_username)
For example, my JSON file looks like this:
{
"twitch": {
"trainwreckstv": {
"online": true,
"streamer_title": "18+ give us 1 f'in w ffs #ad | !twitter | !youtube | !podcast",
"streamer_game": "Slots",
"streamer_uptime": "18:40:53",
"streamer_viewers": "24,922"
},
"mizkif": {
"online": true,
"streamer_title": "(DAY 2) NNN - WHO WAS IN MY ROOM LAST NIGHT? MARIO PARTY LATER | !pobox",
"streamer_game": "Just Chatting",
"streamer_uptime": "4:39:47",
"streamer_viewers": "45,400"
}
}
}
How would I get the value of "streamer_title" starting from a specific base (ex mizkif or trainwreckstv)?
It would be nice if I could possibly do
people.get_value(key, base)
where key is the key and base is the category to start the search from.
For example:
people.get_value('streamer_title','mizkif')
could return:
"(DAY 2) NNN - WHO WAS IN MY ROOM LAST NIGHT? MARIO PARTY LATER | !pobox"
To get the value "streamer_title" from "mizkif", you need to use the [] like with a dictionary.
people["mizkif"]["streamer_title"]
Here's how to implement that "nice method".
Note I changed the name of class to conform to the PEP 8 Naming Conventions.
import json
import os
class DataManager:
BASE_COLLECTIONS_FOLDER = "./data"
def __init__(self, collection_name):
self.collection_name = collection_name
self.collection_file_path = (
os.path.join(self.BASE_COLLECTIONS_FOLDER, self.collection_name+".json"))
self.collection = {}
self.ensure_collection()
self.load_collection()
def get_value(self, key, base): # ADDED METHOD
return self.collection[self.collection_name][base][key]
def ensure_collection(self):
if os.path.isfile(self.collection_file_path):
return
os.makedirs(self.BASE_COLLECTIONS_FOLDER, exist_ok=True)
self.save_collection()
def load_collection(self):
with open(self.collection_file_path, "r", encoding="utf-8") as collection_file:
self.collection = json.load(collection_file) #[self.collection_name]
def save_collection(self):
with open(self.collection_file_path, "w", encoding="utf-8") as collection_file:
json.dump({self.collection_name: self.collection}, collection_file, indent=4)
def write_to_json(self, data, key=None):
if not key:
self.collection = {**self.collection, **data}
else:
self.collection[key] = {**self.collection.get(key, {}), **data}
self.save_collection()
if __name__ == '__main__':
people = DataManager("twitch")
print("people.get_value('streamer_title', 'mizkif') ->")
print(f" {people.get_value('streamer_title', 'mizkif')!r}")
Output:
people.get_value('streamer_title', 'mizkif') ->
'(DAY 2) NNN - WHO WAS IN MY ROOM LAST NIGHT? MARIO PARTY LATER | !pobox'
I'm hoping to use ruamel.yaml to perform some automatic edits against a large human-edited YAML file.
The input file contains merge keys, like so:
foo: &foo
color: red
bar:
name: qux
<<: *foo
If possible, I'd like to preserve the relative ordering of the explicit name key and the << merge key, but it looks like ruamel really wants the merge key to come first. Here's what I get when I round-trip this YAML through ruamel:
foo: &foo
color: red
bar:
<<: *foo
name: qux
Is there any way to tell ruamel to preserve the position of the merge key within this block?
With the tweaking of two lines in the representer for mappings this
can be fixed, the positional information for the merge key was there,
it was just not used. Unfortunately that is a rather large function that
requires a few imports:
import sys
import ruamel.yaml
if ruamel.yaml.version_info < (0, 15, 86):
from ruamel.yaml.nodes import MappingNode, ScalarNode
from ruamel.yaml.comments import comment_attrib, merge_attrib
def represent_mapping(self, tag, mapping, flow_style=None):
value = []
try:
flow_style = mapping.fa.flow_style(flow_style)
except AttributeError:
flow_style = flow_style
try:
anchor = mapping.yaml_anchor()
except AttributeError:
anchor = None
node = MappingNode(tag, value, flow_style=flow_style, anchor=anchor)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = True
# no sorting! !!
try:
comment = getattr(mapping, comment_attrib)
node.comment = comment.comment
if node.comment and node.comment[1]:
for ct in node.comment[1]:
ct.reset()
item_comments = comment.items
for v in item_comments.values():
if v and v[1]:
for ct in v[1]:
ct.reset()
try:
node.comment.append(comment.end)
except AttributeError:
pass
except AttributeError:
item_comments = {}
merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])]
merge_pos = getattr(mapping, merge_attrib, [[0]])[0][0] # <<<<<<<< line added
item_count = 0
if bool(merge_list):
items = mapping.non_merged_items()
else:
items = mapping.items()
for item_key, item_value in items:
item_count += 1
node_key = self.represent_key(item_key)
node_value = self.represent_data(item_value)
item_comment = item_comments.get(item_key)
if item_comment:
assert getattr(node_key, 'comment', None) is None
node_key.comment = item_comment[:2]
nvc = getattr(node_value, 'comment', None)
if nvc is not None: # end comment already there
nvc[0] = item_comment[2]
nvc[1] = item_comment[3]
else:
node_value.comment = item_comment[2:]
if not (isinstance(node_key, ScalarNode) and not node_key.style):
best_style = False
if not (isinstance(node_value, ScalarNode) and not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if ((item_count != 0) or bool(merge_list)) and self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
if bool(merge_list):
# because of the call to represent_data here, the anchors
# are marked as being used and thereby created
if len(merge_list) == 1:
arg = self.represent_data(merge_list[0])
else:
arg = self.represent_data(merge_list)
arg.flow_style = True
value.insert(merge_pos, (ScalarNode(u'tag:yaml.org,2002:merge', '<<'), arg)) # <<<<< line changed
return node
ruamel.yaml.representer.RoundTripRepresenter.represent_mapping = represent_mapping
yaml_str = """\
foo: &foo
color: red
bar:
name: qux
<<: *foo
"""
yaml = ruamel.yaml.YAML()
data = yaml.load(yaml_str)
yaml.dump(data, sys.stdout)
which gives:
foo: &foo
color: red
bar:
name: qux
<<: *foo
The above tries to keep the absolute position, without taking deletion
or inserts of key-value pairs into account.
The above will not patch anything when using the next release of
ruamel.yaml, which will include these changes.
I have a string as
(device
(vfb
(xxxxxxxx)
(xxxxxxxx)
(location 0.0.0.0:5900)
)
)
(device
(console
(xxxxxxxx)
(xxxxxxxx)
(location 80)
)
)
I need to read the location line from "vfb" portion of the string. I have tried to use regular expression like
import re
re.findall(r'device.*?\vfb.*?\(.*?(.*?).*(.*?\))
But it doesn't give me the required output.
It's better to use a parser for problems like this. Fortunately, a parser would be rather trivial in your case:
def parse(source):
def expr(tokens):
t = tokens.pop(0)
if t != '(':
return {'value': t}
key, val = tokens.pop(0), {}
while tokens[0] != ')':
val.update(expr(tokens))
tokens.pop(0)
return {key:val}
tokens = re.findall(r'\(|\)|[^\s()]+', source)
lst = []
while tokens:
lst.append(expr(tokens))
return lst
Given the above snippet, this creates a structure like:
[{'device': {'vfb': {'location': {'value': '0.0.0.0:5900'}, 'xxxxxxxx': {}}}},
{'device': {'console': {'location': {'value': '80'}, 'xxxxxxxx': {}}}}]
Now you can iterate it and fetch whatever you need:
for item in parse(source):
try:
location = item['device']['vfb']['location']['value']
except KeyError:
pass
With that intro from Martijn Pieters, here is a pyparsing approach:
inputdata = """(device
(vfb
(xxxxxxxx)
(xxxxxxxx)
(location 0.0.0.0:5900)
)
)
(device
(console
(xxxxxxxx)
(xxxxxxxx)
(location 80)
)
)"""
from pyparsing import OneOrMore, nestedExpr
# a nestedExpr defaults to reading space-separated words within nested parentheses
data = OneOrMore(nestedExpr()).parseString(inputdata)
print (data.asList())
# recursive search to walk parsed data to find desired entry
def findPath(seq, path):
for s in seq:
if s[0] == path[0]:
if len(path) == 1:
return s[1]
else:
ret = findPath(s[1:], path[1:])
if ret is not None:
return ret
return None
print findPath(data, "device/vfb/location".split('/'))
prints:
[['device', ['vfb', ['xxxxxxxx'], ['xxxxxxxx'], ['location', '0.0.0.0:5900']]],
['device', ['console', ['xxxxxxxx'], ['xxxxxxxx'], ['location', '80']]]]
0.0.0.0:5900
Maybe this gets you started:
In [84]: data = '(device(vfb(xxxxxxxx)(xxxxxxxx)(location 0.0.0.0:5900)))'
In [85]: m = re.search(r"""
.....: vfb
.....: .*
.....: \(
.....: location
.....: \s+
.....: (
.....: [^\)]+
.....: )
.....: \)""", data, flags=re.X)
In [86]: m.group(1)
Out[86]: '0.0.0.0:5900'
I have this code
import json
from pprint import pprint
json_data=open('bookmarks.json')
jdata = json.load(json_data)
pprint (jdata)
json_data.close()
How can I search through it for u'uri': u'http:?
ObjectPath is a library that provides ability to query JSON and nested structures of dicts and lists. For example, you can search for all attributes called "foo" regardless how deep they are by using $..foo.
While the documentation focuses on the command line interface, you can perform the queries programmatically by using the package's Python internals. The example below assumes you've already loaded the data into Python data structures (dicts & lists). If you're starting with a JSON file or string you just need to use load or loads from the json module first.
import objectpath
data = [
{'foo': 1, 'bar': 'a'},
{'foo': 2, 'bar': 'b'},
{'NoFooHere': 2, 'bar': 'c'},
{'foo': 3, 'bar': 'd'},
]
tree_obj = objectpath.Tree(data)
tuple(tree_obj.execute('$..foo'))
# returns: (1, 2, 3)
Notice that it just skipped elements that lacked a "foo" attribute, such as the third item in the list. You can also do much more complex queries, which makes ObjectPath handy for deeply nested structures (e.g. finding where x has y that has z: $.x.y.z). I refer you to the documentation and tutorial for more information.
As json.loads simply returns a dict, you can use the operators that apply to dicts:
>>> jdata = json.load('{"uri": "http:", "foo", "bar"}')
>>> 'uri' in jdata # Check if 'uri' is in jdata's keys
True
>>> jdata['uri'] # Will return the value belonging to the key 'uri'
u'http:'
Edit: to give an idea regarding how to loop through the data, consider the following example:
>>> import json
>>> jdata = json.loads(open ('bookmarks.json').read())
>>> for c in jdata['children'][0]['children']:
... print 'Title: {}, URI: {}'.format(c.get('title', 'No title'),
c.get('uri', 'No uri'))
...
Title: Recently Bookmarked, URI: place:folder=BOOKMARKS_MENU(...)
Title: Recent Tags, URI: place:sort=14&type=6&maxResults=10&queryType=1
Title: , URI: No uri
Title: Mozilla Firefox, URI: No uri
Inspecting the jdata data structure will allow you to navigate it as you wish. The pprint call you already have is a good starting point for this.
Edit2: Another attempt. This gets the file you mentioned in a list of dictionaries. With this, I think you should be able to adapt it to your needs.
>>> def build_structure(data, d=[]):
... if 'children' in data:
... for c in data['children']:
... d.append({'title': c.get('title', 'No title'),
... 'uri': c.get('uri', None)})
... build_structure(c, d)
... return d
...
>>> pprint.pprint(build_structure(jdata))
[{'title': u'Bookmarks Menu', 'uri': None},
{'title': u'Recently Bookmarked',
'uri': u'place:folder=BOOKMARKS_MENU&folder=UNFILED_BOOKMARKS&(...)'},
{'title': u'Recent Tags',
'uri': u'place:sort=14&type=6&maxResults=10&queryType=1'},
{'title': u'', 'uri': None},
{'title': u'Mozilla Firefox', 'uri': None},
{'title': u'Help and Tutorials',
'uri': u'http://www.mozilla.com/en-US/firefox/help/'},
(...)
}]
To then "search through it for u'uri': u'http:'", do something like this:
for c in build_structure(jdata):
if c['uri'].startswith('http:'):
print 'Started with http'
Seems there's a typo (missing colon) in the JSON dict provided by jro.
The correct syntax would be:
jdata = json.load('{"uri": "http:", "foo": "bar"}')
This cleared it up for me when playing with the code.
Functions to search through and print dicts, like JSON.
*made in python 3
Search:
def pretty_search(dict_or_list, key_to_search, search_for_first_only=False):
"""
Give it a dict or a list of dicts and a dict key (to get values of),
it will search through it and all containing dicts and arrays
for all values of dict key you gave, and will return you set of them
unless you wont specify search_for_first_only=True
:param dict_or_list:
:param key_to_search:
:param search_for_first_only:
:return:
"""
search_result = set()
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if key == key_to_search:
if search_for_first_only:
return key_value
else:
search_result.add(key_value)
if isinstance(key_value, dict) or isinstance(key_value, list) or isinstance(key_value, set):
_search_result = pretty_search(key_value, key_to_search, search_for_first_only)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, list) or isinstance(element, set) or isinstance(element, dict):
_search_result = pretty_search(element, key_to_search, search_result)
if _search_result and search_for_first_only:
return _search_result
elif _search_result:
for result in _search_result:
search_result.add(result)
return search_result if search_result else None
Print:
def pretty_print(dict_or_list, print_spaces=0):
"""
Give it a dict key (to get values of),
it will return you a pretty for print version
of a dict or a list of dicts you gave.
:param dict_or_list:
:param print_spaces:
:return:
"""
pretty_text = ""
if isinstance(dict_or_list, dict):
for key in dict_or_list:
key_value = dict_or_list[key]
if isinstance(key_value, dict):
key_value = pretty_print(key_value, print_spaces + 1)
pretty_text += "\t" * print_spaces + "{}:\n{}\n".format(key, key_value)
elif isinstance(key_value, list) or isinstance(key_value, set):
pretty_text += "\t" * print_spaces + "{}:\n".format(key)
for element in key_value:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * (print_spaces + 1) + "{}\n".format(element)
else:
pretty_text += "\t" * print_spaces + "{}: {}\n".format(key, key_value)
elif isinstance(dict_or_list, list) or isinstance(dict_or_list, set):
for element in dict_or_list:
if isinstance(element, dict) or isinstance(element, list) or isinstance(element, set):
pretty_text += pretty_print(element, print_spaces + 1)
else:
pretty_text += "\t" * print_spaces + "{}\n".format(element)
else:
pretty_text += str(dict_or_list)
if print_spaces == 0:
print(pretty_text)
return pretty_text
You can use jsonpipe if you just need the output (and more comfortable with command line):
cat bookmarks.json | jsonpipe |grep uri