Develop Reference

amp azure playback speed python

Im trying to change play back speed in azure amp.
The following is the url generated from azure apis: https://ampdemo.azureedge.net/?url=https://testingmedia-usea.streaming.media.azure.net/bbd51d47-cc1a-4515-bac8-4053040f8c58/ignite.ism/manifest(format=mpd-time-cmaf,filter=filter1)&heuristicprofile=lowlatency
if you check that link there is no playback speed.
I saw the below link but dont know where to apply in python code
https://amp.azure.net/libs/amp/latest/docs/index.html#amp.player.options.playbackspeed
below is my code:
from dotenv import load_dotenv
from azure.identity import DefaultAzureCredential
from azure.mgmt.media import AzureMediaServices
from azure.storage.blob import BlobServiceClient
from azure.mgmt.media.models import (
Asset,
Transform,
TransformOutput,
BuiltInStandardEncoderPreset,
Job,
JobInputAsset,
JobOutputAsset,
OnErrorType,
Priority,
StreamingLocator,
AssetFilter,
PresentationTimeRange,
)
import os
import random
#Timer for checking job progress
import time
import requests
#Get environment variables
load_dotenv()
default_credential = DefaultAzureCredential(exclude_shared_token_cache_credential=True)
# Get the environment variables SUBSCRIPTIONID, RESOURCEGROUP and ACCOUNTNAME
subscription_id = os.getenv('SUBSCRIPTIONID')
resource_group = os.getenv('RESOURCEGROUP')
account_name = os.getenv('ACCOUNTNAME')
# The file you want to upload. For this example, put the file in the same folder as this script.
# The file ignite.mp4 has been provided for you.
source_file = "https://testingmedia.blob.core.windows.net/data/ignite.mp4"
#url = requests.get(source_file)
# This is a random string that will be added to the naming of things so that you don't have to keep doing this during testing
uniqueness = "streamAssetFilters-" + str(random.randint(0,9999))
# Change this to your specific streaming endpoint name if not using "default"
streaming_endpoint_name = "default"
# Set the attributes of the input Asset using the random number
in_asset_name = 'inputassetName' + uniqueness
in_alternate_id = 'inputALTid' + uniqueness
in_description = 'inputdescription' + uniqueness
# Create an Asset object
# The asset_id will be used for the container parameter for the storage SDK after the asset is created by the AMS client.
in_asset = Asset(alternate_id=in_alternate_id, description=in_description)
# Set the attributes of the output Asset using the random number
out_asset_name = 'outputassetName' + uniqueness
out_alternate_id = 'outputALTid' + uniqueness
out_description = 'outputdescription' + uniqueness
# Create an output asset object
out_asset = Asset(alternate_id=out_alternate_id, description=out_description)
# The AMS Client
print("Creating AMS Client")
client = AzureMediaServices(default_credential, subscription_id)
# Create an input Asset
print(f"Creating input asset {in_asset_name}")
input_asset = client.assets.create_or_update(resource_group, account_name, in_asset_name, in_asset)
# An AMS asset is a container with a specific id that has "asset-" prepended to the GUID.
# So, you need to create the asset id to identify it as the container
# where Storage is to upload the video (as a block blob)
in_container = 'asset-' + input_asset.asset_id
# create an output Asset
print(f"Creating output asset {out_asset_name}")
output_asset = client.assets.create_or_update(resource_group, account_name, out_asset_name, out_asset)
### Use the Storage SDK to upload the video ###
print(f"Uploading the file {source_file}")
blob_service_client = BlobServiceClient.from_connection_string(os.getenv('STORAGEACCOUNTCONNECTION'))
blob_client = blob_service_client.get_blob_client(in_container, "ignite.mp4")
# working_dir = os.getcwd() + "\Media"
# print(working_dir)
# print(f"Current working directory: {working_dir}")
# upload_file_path = os.path.join(working_dir, source_file)
# print(upload_file_path,"####")
# WARNING: Depending on where you are launching the sample from, the path here could be off, and not include the BasicEncoding folder.
# Adjust the path as needed depending on how you are launching this python sample file.
# Upload the video to storage as a block blob
#with open(url, "rb") as data:
blob_client.upload_blob_from_url(source_file)
transform_name = 'ContentAwareEncodingAssetFilters'
# Create a new Standard encoding Transform for Built-in Copy Codec
print(f"Creating Encoding transform named: {transform_name}")
# For this snippet, we are using 'BuiltInStandardEncoderPreset'
transform_output = TransformOutput(
preset=BuiltInStandardEncoderPreset(
preset_name="ContentAwareEncoding"
),
# What should we do with the job if there is an error?
on_error=OnErrorType.STOP_PROCESSING_JOB,
# What is the relative priority of this job to others? Normal, high or low?
relative_priority=Priority.NORMAL
)
print("Creating encoding transform...")
# Adding transform details
my_transform = Transform()
my_transform.description="Transform with Asset filters"
my_transform.outputs = [transform_output]
print(f"Creating transform {transform_name}")
transform = client.transforms.create_or_update(
resource_group_name=resource_group,
account_name=account_name,
transform_name=transform_name,
parameters=my_transform)
print(f"{transform_name} created (or updated if it existed already). ")
job_name = 'ContentAwareEncodingAssetFilters'+ uniqueness
print(f"Creating custom encoding job {job_name}")
files = (source_file)
# Create Job Input and Ouput Assets
input = JobInputAsset(asset_name=in_asset_name)
outputs = JobOutputAsset(asset_name=out_asset_name)
# Create the job object and then create transform job
the_job = Job(input=input, outputs=[outputs])
job: Job = client.jobs.create(resource_group, account_name, transform_name, job_name, parameters=the_job)
# Check job state
job_state = client.jobs.get(resource_group, account_name, transform_name, job_name)
# First check
print("First job check")
print(job_state.state)
# Check the state of the job every 10 seconds. Adjust time_in_seconds = <how often you want to check for job state>
def countdown(t):
while t:
mins, secs = divmod(t, 60)
timer = '{:02d}:{:02d}'.format(mins, secs)
print(timer, end="\r")
time.sleep(1)
t -= 1
job_current = client.jobs.get(resource_group, account_name, transform_name, job_name)
if(job_current.state == "Finished"):
print(job_current.state)
# TODO: Download the output file using blob storage SDK
return
if(job_current.state == "Error"):
print(job_current.state)
# TODO: Provide Error details from Job through API
return
else:
print(job_current.state)
countdown(int(time_in_seconds))
time_in_seconds = 10
countdown(int(time_in_seconds))
print(f"Creating locator for streaming...")
# Publish the output asset for streaming via HLS or DASH
locator_name = f"locator-{uniqueness}"
# Create the Asset filters
print("Creating an asset filter...")
asset_filter_name = 'filter1'
# Create the asset filter
asset_filter = client.asset_filters.create_or_update(
resource_group_name=resource_group,
account_name=account_name,
asset_name=out_asset_name,
filter_name=asset_filter_name,
parameters=AssetFilter(
# In this sample, we are going to filter the manifest by the time range of the presentation using the default timescale.
# You can adjust these settings for your own needs. Not that you can also control output tracks, and quality levels with a filter.
tracks=[],
# start_timestamp = 100000000 and end_timestamp = 300000000 using the default timescale will generate
# a play-list that contains fragments from between 10 seconds and 30 seconds of the VoD presentation.
# If a fragment straddles the boundary, the entire fragment will be included in the manifest.
presentation_time_range=PresentationTimeRange(start_timestamp=100000000, end_timestamp=300000000)
)
)
if asset_filter:
print(f"The asset filter ({asset_filter_name}) was successfully created.")
print()
else:
raise ValueError("There was an issue creating the asset filter.")
if output_asset:
streaming_locator = StreamingLocator(asset_name=out_asset_name, streaming_policy_name="Predefined_DownloadAndClearStreaming",filters=list(asset_filter_name.split(" ")))
locator = client.streaming_locators.create(
resource_group_name=resource_group,
account_name=account_name,
streaming_locator_name=locator_name,
parameters=streaming_locator
)
if locator:
print(f"The streaming locator {locator_name} was successfully created!")
else:
raise Exception(f"Error while creating streaming locator {locator_name}")
if locator.name:
hls_format = "format=m3u8-cmaf"
dash_format = "format=mpd-time-cmaf"
# Get the default streaming endpoint on the account
streaming_endpoint = client.streaming_endpoints.get(
resource_group_name=resource_group,
account_name=account_name,
streaming_endpoint_name=streaming_endpoint_name
)
if streaming_endpoint.resource_state != "Running":
print(f"Streaming endpoint is stopped. Starting endpoint named {streaming_endpoint_name}")
client.streaming_endpoints.begin_start(resource_group, account_name, streaming_endpoint_name)
basename_tup = os.path.splitext(source_file) # Extracting the filename and extension
path_extension = basename_tup[1] # Setting extension of the path
manifest_name = os.path.basename(source_file).replace(path_extension, "")
print(f"The manifest name is: {manifest_name}")
manifest_base = f"https://{streaming_endpoint.host_name}/{locator.streaming_locator_id}/{manifest_name}.ism/manifest"
hls_manifest = ""
if asset_filter_name is None:
hls_manifest = f'{manifest_base}({hls_format})'
else:
hls_manifest = f'{manifest_base}({hls_format},filter={asset_filter_name})'
print(f"The HLS (MP4) manifest URL is: {hls_manifest}")
print("Open the following URL to playback the live stream in an HLS compliant player (HLS.js, Shaka, ExoPlayer) or directly in an iOS device")
print({hls_manifest})
print()
dash_manifest = ""
if asset_filter_name is None:
dash_manifest = f'{manifest_base}({dash_format})'
else:
dash_manifest = f'{manifest_base}({dash_format},filter={asset_filter_name})'
print(f"The DASH manifest URL is: {dash_manifest}")
print("Open the following URL to playback the live stream from the LiveOutput in the Azure Media Player")
print(f"https://ampdemo.azureedge.net/?url={dash_manifest}&heuristicprofile=lowlatency")
print()
else:
raise ValueError("Locator was not created or Locator name is undefined.")

There's an example on https://amp.azure.net/libs/amp/latest/samples/dynamic_playback_speed.html for how to use playback speed. This is also available at https://github.com/Azure-Samples/azure-media-player-samples/blob/master/html/dynamic_playback_speed.html.

Running Google Cloud DocumentAI sample code on Python returned the error 503

I am trying the example from the Google repo:
https://github.com/googleapis/python-documentai/blob/HEAD/samples/snippets/quickstart_sample.py
I have an error:
metadata=[('x-goog-request-params', 'name=projects/my_proj_id/locations/us/processors/my_processor_id'), ('x-goog-api-client', 'gl-python/3.8.10 grpc/1.38.1 gax/1.30.0 gapic/1.0.0')]), last exception: 503 DNS resolution failed for service: https://us-documentai.googleapis.com/v1/
My full code:
from google.cloud import documentai_v1 as documentai
import os
# TODO(developer): Uncomment these variables before running the sample.
project_id= '123456789'
location = 'us' # Format is 'us' or 'eu'
processor_id = '1a23345gh823892' # Create processor in Cloud Console
file_path = 'document.jpg'
os.environ['GRPC_DNS_RESOLVER'] = 'native'
def quickstart(project_id: str, location: str, processor_id: str, file_path: str):
# You must set the api_endpoint if you use a location other than 'us', e.g.:
opts = {}
if location == "eu":
opts = {"api_endpoint": "eu-documentai.googleapis.com"}
client = documentai.DocumentProcessorServiceClient(client_options=opts)
# The full resource name of the processor, e.g.:
# projects/project-id/locations/location/processor/processor-id
# You must create new processors in the Cloud Console first
name = f"projects/{project_id}/locations/{location}/processors/{processor_id}:process"
# Read the file into memory
with open(file_path, "rb") as image:
image_content = image.read()
document = {"content": image_content, "mime_type": "image/jpeg"}
# Configure the process request
request = {"name": name, "raw_document": document}
result = client.process_document(request=request)
document = result.document
document_pages = document.pages
# For a full list of Document object attributes, please reference this page: https://googleapis.dev/python/documentai/latest/_modules/google/cloud/documentai_v1beta3/types/document.html#Document
# Read the text recognition output from the processor
print("The document contains the following paragraphs:")
for page in document_pages:
paragraphs = page.paragraphs
for paragraph in paragraphs:
print(paragraph)
paragraph_text = get_text(paragraph.layout, document)
print(f"Paragraph text: {paragraph_text}")
def get_text(doc_element: dict, document: dict):
"""
Document AI identifies form fields by their offsets
in document text. This function converts offsets
to text snippets.
"""
response = ""
# If a text segment spans several lines, it will
# be stored in different text segments.
for segment in doc_element.text_anchor.text_segments:
start_index = (
int(segment.start_index)
if segment in doc_element.text_anchor.text_segments
else 0
)
end_index = int(segment.end_index)
response += document.text[start_index:end_index]
return response
def main ():
quickstart (project_id = project_id, location = location, processor_id = processor_id, file_path = file_path)
if __name__ == '__main__':
main ()
FYI, on the Google Cloud website it stated that the endpoint is:
https://us-documentai.googleapis.com/v1/projects/123456789/locations/us/processors/1a23345gh823892:process
I can use the web interface to run DocumentAI so it is working. I just have the problem with Python code.
Any suggestion is appreciated.

I would suspect the GRPC_DNS_RESOLVER environment variable to be the root cause. Did you try with the corresponding line commented out? Why was it added in your code?

Python: Masking Named Entities in Email Text

I have created a python script to extract named entities as follows:
# set java path
java_path = r'C:/Program Files/Java/jre1.8.0_161/bin/java.exe'
os.environ['JAVAHOME'] = java_path
# initialize NER tagger
sn = StanfordNERTagger('C:/Users/Parag/Documents/stanford-ner-2018-10-16/stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz',
path_to_jar='C:/Users/Parag/Documents/stanford-ner-2018-10-16/stanford-ner-2018-10-16/stanford-ner.jar')
# tag named entities
ner_tagged_sentences = [sn.tag(sent.split()) for sent in dataset_unseen['Text']]
dataset_unseen['Text'] = dataset_unseen.apply(Detectner,axis=1)
# extract all named entities
named_entities = []
for sentence in ner_tagged_sentences:
temp_entity_name = ''
temp_named_entity = None
for term, tag in sentence:
if tag != 'O':
temp_entity_name = ' '.join([temp_entity_name, term]).strip()
temp_named_entity = (temp_entity_name, tag)
else:
if temp_named_entity:
named_entities.append(temp_named_entity)
temp_entity_name = ''
temp_named_entity = None
entity_frame = pd.DataFrame(named_entities, columns=['Entity Name', 'Entity Type'])
entity_frame.head()
** Output **
Entity Name Entity Type Frequency
ABC Farms ORGANIZATION 5
Freddy Hill Lane ORGANIZATION 3
North Lane Thames ORGANIZATION 2
Now I want to mask these named entities with a pattern like "######" to follow GDPR regulations by hiding customer sensitive information.
I have attempted options like:
Applying for loop on original data frame - check Text for named entities present in named entity data frame - mask named entity with '#####'.
Define a function for masking named entities in Text:
def Detectner(row):
ner_tagged_sentences = [sn.tag(sent.split()) for sent in row['Text']]
results = ner_tagged_sentences.sub('**********',row['Text'])
return results
dataset_unseen['Text'] = dataset_unseen.apply(Detectner,axis=1)
But I get Following Error:
AttributeError: ("'list' object has no attribute 'sub'", 'occurred at index 0')
How can I extract and mask named entities in Text. Any improvement to this code is highly apprciated !

When you make the tagged sentences, you are creating a list in the line
ner_tagged_sentences = [sn.tag(sent.split()) for sent in row['Text']]
The type of ner_tagged_sentences is list which has no sub method.
You can try multiple things to achieve your goal to make documents anonymous:
Replace tokens with non-O tags with something (token-level)
Replace Named Entity Text directly in document (string-level)
It seems like you are trying to do number (2)

transferring rdf to 4store

actually I have a code named rdf.py that generates rdf code ..what I want to do is to directly move that file in 4store.. I have stored the entire code in a variable and want to directly pass that variable to 4store.. is it possible?
the code of rdf.py is below.
rdf_code contains the entire rdf code that is generated
import rdflib
from rdflib.events import Dispatcher, Event
from rdflib.graph import ConjunctiveGraph as Graph
from rdflib import plugin
from rdflib.store import Store, NO_STORE, VALID_STORE
from rdflib.namespace import Namespace
from rdflib.term import Literal
from rdflib.term import URIRef
from tempfile import mkdtemp
from gstudio.models import *
from objectapp.models import *
from reversion.models import Version
from optparse import make_option
def get_nodetype(name):
"""
returns the model the id belongs to.
"""
try:
"""
ALGO: get object id, go to version model, return for the given id.
"""
node = NID.objects.get(title=str(name))
# Retrieving only the relevant tupleset for the versioned objects
vrs = Version.objects.filter(type=0 , object_id=node.id)
# Returned value is a list, so splice it .
vrs = vrs[0]
except Error:
return "The item was not found."
return vrs.object._meta.module_name
def rdf_description(name, notation='xml' ):
"""
Function takes title of node, and rdf notation.
"""
valid_formats = ["xml", "n3", "ntriples", "trix"]
default_graph_uri = "http://gstudio.gnowledge.org/rdfstore"
configString = "/var/tmp/rdfstore"
# Get the Sleepycat plugin.
store = plugin.get('IOMemory', Store)('rdfstore')
# Open previously created store, or create it if it doesn't exist yet
graph = Graph(store="IOMemory",
identifier = URIRef(default_graph_uri))
path = mkdtemp()
rt = graph.open(path, create=False)
if rt == NO_STORE:
#There is no underlying Sleepycat infrastructure, create it
graph.open(path, create=True)
else:
assert rt == VALID_STORE, "The underlying store is corrupt"
# Now we'll add some triples to the graph & commit the changes
# rdflib = Namespace('http://sbox.gnowledge.org/gstudio/')
graph.bind("gstudio", "http://gnowledge.org/")
exclusion_fields = ["id", "rght", "node_ptr_id", "image", "lft", "_state", "_altnames_cache", "_tags_cache", "nid_ptr_id", "_mptt_cached_fields"]
node_type=get_nodetype(name)
if (node_type=='gbobject'):
node=Gbobject.objects.get(title=name)
elif (node_type=='objecttype'):
node=Objecttype.objects.get(title=name)
elif (node_type=='metatype'):
node=Metatype.objects.get(title=name)
elif (node_type=='attributetype'):
node=Attributetype.objects.get(title=name)
elif (node_type=='relationtype'):
node=Relationtype.objects.get(title=name)
elif (node_type=='attribute'):
node=Attribute.objects.get(title=name)
elif (node_type=='complement'):
node=Complement.objects.get(title=name)
elif (node_type=='union'):
node=Union.objects.get(title=name)
elif (node_type=='intersection'):
node=Intersection.objects.get(title=name)
elif (node_type=='expression'):
node=Expression.objects.get(title=name)
elif (node_type=='processtype'):
node=Processtype.objects.get(title=name)
elif (node_type=='systemtype'):
node=Systemtype.objects.get(title=name)
node_url=node.get_absolute_url()
site_add= node.sites.all()
a = site_add[0]
host_name =a.name
#host_name=name
link='http://'
#Concatenating the above variables will give the url address.
url_add=link+host_name+node_url
rdflib = Namespace(url_add)
# node=Objecttype.objects.get(title=name)
node_dict=node.__dict__
subject=str(node_dict['id'])
for key in node_dict:
if key not in exclusion_fields:
predicate=str(key)
pobject=str(node_dict[predicate])
graph.add((rdflib[subject], rdflib[predicate], Literal(pobject)))
rdf_code= graph.serialize(format=notation)
# print out all the triples in the graph
for subject, predicate, object in graph:
print subject, predicate, object
graph.commit()
print rdf_code
graph.close()
can I directly pass the rdf_code to 4store...if yes then how?

The simplest way to do this is to transform that graph into ntriples and send it to http://yourhost:port/data/GRAPH_URI. If you do an HTTP POST then the triples will be appended to the existing graph represented by GRAPH_URI. If you do a HTTP PUT then the current graph will be replaced. If the graph does not exist then it will be created no matter if you POST or PUT.
Taking this function as example:
def assert4s(data,epr,graph,contenttype,flush=False):
try:
params = urllib.urlencode({'graph': graph,
'data': data,
'mime-type' : contenttype })
opener = urllib2.build_opener(urllib2.HTTPHandler)
request = urllib2.Request(epr,params)
request.get_method = lambda: ('PUT' if flush else 'POST')
url = opener.open(request)
return url.read()
except Exception, e:
raise e
If you had the following data:
triples = """<a> <b> <c> .
<d> <e> <f> .
"""
You can do the following call:
assert4s(triples,
"http://yourhost:port/data/",
"http://some.org/graph/id",
"application/x-turtle")
Edit
My previous answer assumed you were using the 4s-httpd server. You can start the SPARQL server in 4store with the following command 4s-httpd -p PORT kb_name. Once you have this running, you can use the following services for:
http://localhost:port/sparql/ to submit queries
http://localhost:port/data/ to PUT or POST data files.
http://localhost:port/update/ to submit SPARQL updates queries.
The 4store SPARQLServer documentation is quite complete.

Using Boto to find to which device and EBS Volume is mounted

How do I find to which device an EBS Volume is mounted with Python Boto v2.0?
boto.ec2.Volume has some interesting properies like attachment_state and volume_state. But are there any functions for device mapping?
boto.manage.volume has get_device(self, params) but requires a CommandLineGetter.
Any pointers on how to proceed or some samples of using boto.manage?

I believe attach_data.device is what your looking for. part of volume.
Heres an example, not sure if this is the best way, but it outputs volumeid, instanceid, and attachment_data something like:
Attached Volume ID - Instance ID - Device Name
vol-12345678 - i-ab345678 - /dev/sdp
vol-12345678 - i-ab345678 - /dev/sda1
vol-12345678 - i-cd345678 - /dev/sda1
import boto
ec2 = boto.connect_ec2()
res = ec2.get_all_instances()
instances = [i for r in res for i in r.instances]
vol = ec2.get_all_volumes()
def attachedvolumes():
print 'Attached Volume ID - Instance ID','-','Device Name'
for volumes in vol:
if volumes.attachment_state() == 'attached':
filter = {'block-device-mapping.volume-id':volumes.id}
volumesinstance = ec2.get_all_instances(filters=filter)
ids = [z for k in volumesinstance for z in k.instances]
for s in ids:
print volumes.id,'-',s.id,'-',volumes.attach_data.device
# Get a list of unattached volumes
def unattachedvolumes():
for unattachedvol in vol:
state = unattachedvol.attachment_state()
if state == None:
print unattachedvol.id, state
attachedvolumes()
unattachedvolumes()

It isn't clear if you're running this from the instance itself or externally. If the latter, you will not need the metadata call. Just supply the instance id.
from boto.ec2.connection import EC2Connection
from boto.utils import get_instance_metadata
conn = EC2Connection()
m = get_instance_metadata()
volumes = [v for v in conn.get_all_volumes() if v.attach_data.instance_id == m['instance-id']]
print(volumes[0].attach_data.device)
Note that an instance may have multiple volumes, so robust code won't assume there's a single device.

If you also want the block device mappings (in linux, the local device name of the EBS volume), you can also use EC2Connection.get_instance_attribute to retrieve a list of the local device names and their corresponding EBS objects:
def get_block_device_mapping(instance_id):
return conn.get_instance_attribute(
instance_id=instance_id,
attribute='blockDeviceMapping'
)['blockDeviceMapping']
This will return a dictionary with local device names as keys, and EBS objects as values (from which you can get all sorts of things like the volume-id).

The best way I've found is to get all resources in one region at a time and associate them yourself:
#!/usr/bin/env python2
import boto.ec2
REGION = 'us-east'
CONN = boto.ec2.connect_to_region(REGION)
def main():
volumes = conn.get_all_volumes()
for volume in volumes:
print volume
# Match to an instance id
print volume.attach_data.instance_id
# # Object attributes:
# print volume.__dict__
# # Object methods:
# print(dir(volume))
if __name__ == '__main__':
main()