I'm using python to access elasticsearch cluster. Now I want to backup my index by using snapshot.
The most difficult thing is that: the python-elasticsearch's doc just give me a API description. there is no example to show me how to create snapshot. I tried some parameters, but failed. Can anyone give a snapshot example of elastic-search by using python? The following is my code:
from elasticsearch import Elasticsearch
es = Elasticsearch()
snapshot_body = {
"type": "url",
"settings": {
"url": "http://download.elasticsearch.org/definitiveguide/sigterms_demo/"
}
}
body = {"snapshot": snapshot_body}
es.snapshot.create_repository(repository='test', body=body)
Your repository creation is almost correct, you don't need the line body = {"snapshot": snapshot_body}, simply create your repository like this:
es.snapshot.create_repository(repository='test', body=snapshot_body)
Now in order to create a snapshot, all you have to do is this:
es.snapshot.create(repository='test', snapshot='my_snapshot')
If you want to store only a few indices and not all you can also provide a body like this:
index_body = {
"indices": "index_1,index_2"
}
es.snapshot.create(repository='test', snapshot='my_snapshot', body=index_body)
Save the following sample Python code as a Python file, such as register-repo.py. The client requires the AWS SDK for Python (Boto3), requests and requests-aws4auth packages. The client contains commented-out examples for other snapshot operations.
import boto3
import requests
from requests_aws4auth import AWS4Auth
host = '' # include https:// and trailing /
region = '' # e.g. us-west-1
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)
# Register repository
path = '_snapshot/my-snapshot-repo-name' # the Elasticsearch API endpoint
url = host + path
payload = {
"type": "s3",
"settings": {
"bucket": "s3-bucket-name",
# "endpoint": "s3.amazonaws.com", # for us-east-1
"region": "us-west-1", # for all other regions
"role_arn": "arn:aws:iam::123456789012:role/TheSnapshotRole"
}
}
headers = {"Content-Type": "application/json"}
r = requests.put(url, auth=awsauth, json=payload, headers=headers)
print(r.status_code)
print(r.text)
# # Take snapshot
#
# path = '_snapshot/my-snapshot-repo/my-snapshot'
# url = host + path
#
# r = requests.put(url, auth=awsauth)
#
# print(r.text)
#
# # Delete index
#
# path = 'my-index'
# url = host + path
#
# r = requests.delete(url, auth=awsauth)
#
# print(r.text)
#
# # Restore snapshot (all indices except Kibana and fine-grained access control)
#
# path = '_snapshot/my-snapshot-repo/my-snapshot/_restore'
# url = host + path
#
# payload = {
# "indices": "-.kibana*,-.opendistro_security",
# "include_global_state": false
# }
#
# headers = {"Content-Type": "application/json"}
#
# r = requests.post(url, auth=awsauth, json=payload, headers=headers)
#
# # Restore snapshot (one index)
#
# path = '_snapshot/my-snapshot-repo/my-snapshot/_restore'
# url = host + path
#
# payload = {"indices": "my-index"}
#
# headers = {"Content-Type": "application/json"}
#
# r = requests.post(url, auth=awsauth, json=payload, headers=headers)
#
# print(r.text)
DONT USE THIS IN US EAST 1 then you have to use this
Important
If the S3 bucket is in the us-east-1 region, you must use "endpoint": "s3.amazonaws.com" instead of "region": "us-east-1".
To enable server-side encryption with S3-managed keys for the snapshot repository, add "server_side_encryption": true to the "settings" JSON.
https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains-snapshots.html#es-managedomains-snapshot-registerdirectory
Related
I am trying to upload reel with Graph Api using Python.
I'm getting an error every time I try to upload video.
Error :
{"debug_info":{"retriable":false,"type":"NotAuthorizedError","message":"User not authorized to perform this request"}}
Note: I have given every possible permission to my app and page.
code:
import requests
import os
import json
Title ="Title of the video"
title = Title
description = title
source = f"F:\proj_ytTofb\downloads\{Title}.mp4"
files = {'source': open(source, 'rb')}
file_size = os.path.getsize(source)
print("File Size is :", file_size, "bytes")
def Initialize():
url = f"https://graph.facebook.com/v13.0/{page_id}/video_reels?upload_phase=start"
payload = {
'access_token': token,
}
r = requests.post(url, data = payload)
return r.json()["video_id"]
video_id=Initialize()
print(video_id)
def Upload():
url = f"https://rupload.facebook.com/video-upload/v15.0/{video_id}"
payload ={
'access_token': token,
'offset': 0,
'file_size': file_size,
}
r = requests.post(url, files=files, data=payload)
return r.text
print(Upload())
output: {"debug_info":{"retriable":false,"type":"NotAuthorizedError","message":"User not authorized to perform this request"}}
Your Upload code seem bit wrong if you refer on documentation
def Upload(vidid, size, filedata):
url = f"https://rupload.facebook.com/video-upload/v13.0/{vidid}"
payloadUp = {
'Authorization': 'OAuth ' + page_access_token,
'offset': "0",
'file_size': str(size),
}
print(payloadUp)
r = requests.post(url, data=filedata, headers=payloadUp)
return r.text
with parameter like this
files = {'source': open(mp4_path, 'rb')}
file_size = os.path.getsize(mp4_path)
and then you called it like this
Upload(video_id, file_size, files)
Note: I successfully upload it on fb reels and published it, but I dont what happen the video failed to convert without error notice.
I have this API GET request that works fine with the full address:
# Transactions
transaction_url = "https://api.whale-alert.io/v1/transaction/ethereum/0015286d8642f0e0553b7fefa1c168787ae71173cbf82ec2f2a1b2e0ffee72b2"
transaction_querystring = {
"api_key":"APIKEY"
}
transaction_response = requests.request("GET", transaction_url, params=transaction_querystring)
print(transaction_response)
print(transaction_response.text)
but, when I try to pass the variables as headers:
# Transactions
transaction_url = "https://api.whale-alert.io/v1/transaction"
transaction_querystring = {
"api_key":"APIKEY"
}
transaction_headers = {
'blockchain': "ethereum",
'hash': "0015286d8642f0e0553b7fefa1c168787ae71173cbf82ec2f2a1b2e0ffee72b2"
}
transaction_response = requests.request("GET", transaction_url, headers=transaction_headers, data=transaction_querystring, )
print(transaction_response)
print(transaction_response.text)
It won't work:
<Response [404]
Not Found
made it work with {0}".format(startime_unix)
I am working on a sandbox environment and I am trying to download the report based on the https://advertising.amazon.com/API/docs/v2/reference/reports
The issue is that the report which is downloaded is empty. Doesn't contain any data inside. How do we download the report from amazon advertising api?
I follow this steps as described: https://gist.github.com/dbrent-amazon/ca396a63c1670ee0ec83aad26b0ce55b
here's a script that works for me using python and requests, make sure to:
create a campaign, adGroupd and keywords
create reports with the right metrics
make sure you have actual data to view in the report
make sure reportDate is correct
import requests
version = 'v2'
advertise = 'sp'
headers = {
"Authorization": f"Bearer {token.access}",
"Amazon-Advertising-API-ClientId": AmazonSecurityProfile.ClientID,
"Content-Type": "application/json",
}
class urls:
class api:
test = 'https://advertising-api-test.amazon.com'
# create report
recordType = "keywords"
r = requests.post(
f'{urls.api.test}/{version}/{advertise}/{recordType}/report',
json={
# "campaignType": "sponsoredProducts",
"segment": "query",
"reportDate": '20201025', #YYYYMMDD
"metrics": ",".join([
"campaignName",
"campaignId",
"campaignStatus",
"campaignBudget",
"clicks",
"cost",
"attributedConversions1d",
"attributedConversions7d",
"attributedConversions1dSameSKU",
"attributedConversions7dSameSKU",
"attributedUnitsOrdered1d",
"attributedUnitsOrdered7d",
"attributedSales1d",
"attributedSales7d",
"attributedSales1dSameSKU",
"attributedSales7dSameSKU",
"attributedUnitsOrdered1dSameSKU",
"attributedUnitsOrdered7dSameSKU",
"adGroupName",
"adGroupId",
"keywordText",
"keywordId",
"matchType",
"impressions",
]),
},
headers=headers,
)
r.raise_for_status()
r = r.json()
print(r)
reportId = r["reportId"]
while r['status'] == 'IN_PROGRESS':
r = requests.get(
f'{urls.api.test}/{version}/reports/{reportId}',
headers=headers,
)
r = r.json()
print(r)
assert r['status'] == 'SUCCESS'
r = requests.get(
r["location"],
headers=headers,
)
print(r)
I am working with an API that doesn't have all the information I need in a single call, and I need to the project code it came from into the call that I am making. Right now it appends the project data to the list, but I really need it to be part of the original call. Here is my output now:
[{"committer_email": "justin.m.boucher#example.com", "short_id": "981147b9", "title": "Added .gitignore", "author_email": "justin.m.boucher#example.com", "authored_date": "2017-08-29T08:31:11.000-07:00", "created_at": "2017-08-29T08:31:11.000-07:00", "author_name": "Justin Boucher", "parent_ids": [], "committed_date": "2017-08-29T08:31:11.000-07:00", "message": "Added .gitignore\n", "committer_name": "Justin Boucher", "id": "981147b905913a60796283ce10f915c53679df49"}, {"project_id": "2"}]
Here is the output I want to achieve:
[{"project_id": "2", "committer_email": "justin.m.boucher#example.com", "short_id": "981147b9", "title": "Added .gitignore", "author_email": "justin.m.boucher#example.com", "authored_date": "2017-08-29T08:31:11.000-07:00", "created_at": "2017-08-29T08:31:11.000-07:00", "author_name": "Justin Boucher", "parent_ids": [], "committed_date": "2017-08-29T08:31:11.000-07:00", "message": "Added .gitignore\n", "committer_name": "Justin Boucher", "id": "981147b905913a60796283ce10f915c53679df49"}]
Here is my code so far:
get_commits.py:
import gitlab
import json
gitlab = gitlab.Gitlab()
projects = gitlab.getProjectID()
for i in projects:
api_str = '/projects/' + str(i) + '/repository/commits'
connect = gitlab.connectAPI(apiCall=api_str)
data = json.dumps(connect)
# Append project id to json, since it isn't created
# in the commits from Gitlab
commit = json.loads(data)
commit.append({'project_id': str(i)})
# make it pretty again for Splunk to read
commit = json.dumps(commit)
print commit
gitlab.py
import os
import ConfigParser
import requests
import json
# Setup Splunk Environment
APPNAME = 'splunk_gitlab'
CONFIG = 'appconfig.conf'
SPLUNK_HOME = os.environ['SPLUNK_HOME']
parser = ConfigParser.SafeConfigParser()
class Gitlab():
# # Load Settings
# parser.read(SPLUNK_HOME + '/etc/apps/' + APPNAME + '/local/' + CONFIG)
# if parser.has_section('Authentication'):
# pass
# else:
# parser.read(SPLUNK_HOME + '/etc/apps/' + APPNAME + '/default/' + CONFIG)
#
# GITLAB_URL = parser.get('Authentication', 'GITLAB_URL')
# API_KEY = parser.get('Authentication', 'API_KEY')
# Used for testing only
GITLAB_URL = 'http://<my_address>'
API_KEY = '<my_key>'
API_SERVER = GITLAB_URL + '/api/v4'
# Place api call to retrieve data
def connectAPI(self, apiCall='/projects'):
headers = {
'PRIVATE-TOKEN': self.API_KEY
}
final_url = self.API_SERVER + apiCall
resp = requests.get(final_url, headers=headers)
status_code = resp.status_code
resp = resp.json()
if status_code == 200:
return resp
else:
raise Exception("Something went wrong requesting (%s): %s" % (
resp['errors'][0]['errorType'], resp['errors'][0]['message']))
def getProjectID(self):
connect = self.connectAPI(apiCall='/projects')
data = json.dumps(connect)
projects = json.loads(data)
project_list = []
for i in projects:
project_list.append(i['id'])
return project_list
If you want to add a new element to the first dictionary in the list instead of appending a new dictionary to the list, try using assignment instead of append.
commit[0]['project_id'] = str(i)
I am able to send text to Mattermost channel through incoming webhooks
import requests, json
URL = 'http://chat.something.com/hooks/1pgrmsj88qf5jfjb4eotmgfh5e'
payload = {"channel": "general", "text": "some text"}
r = requests.post(URL, data=json.dumps(payload))
this code simplly post text. I could not find a way to post file to channel. Suppose I want to post file located at /home/alok/Downloads/Screenshot_20170217_221447.png. If anyone know please share.
You can't currently attach files using the Incoming Webhooks API. You would need to use the Mattermost Client API to make a post with files attached to it.
Here's an example of how you could achieve that (using Mattermost API v3 for Mattermost >= 3.5)
SERVER_URL = "http://chat.example.com/"
TEAM_ID = "team_id_goes_here"
CHANNEL_ID = "channel_id_goes_here"
USER_EMAIL = "you#example.com"
USER_PASS = "password123"
FILE_PATH = '/home/user/thing_to_upload.png'
import requests, json, os
# Login
s = requests.Session() # So that the auth cookie gets saved.
s.headers.update({"X-Requested-With": "XMLHttpRequest"}) # To stop Mattermost rejecting our requests as CSRF.
l = s.post(SERVER_URL + 'api/v3/users/login', data = json.dumps({'login_id': USER_EMAIL, 'password': USER_PASS}))
USER_ID = l.json()["id"]
# Upload the File.
form_data = {
"channel_id": ('', CHANNEL_ID),
"client_ids": ('', "id_for_the_file"),
"files": (os.path.basename(FILE_PATH), open(FILE_PATH, 'rb')),
}
r = s.post(SERVER_URL + 'api/v3/teams/' + TEAM_ID + '/files/upload', files=form_data)
FILE_ID = r.json()["file_infos"][0]["id"]
# Create a post and attach the uploaded file to it.
p = s.post(SERVER_URL + 'api/v3/teams/' + TEAM_ID + '/channels/' + CHANNEL_ID + '/posts/create', data = json.dumps({
'user_id': USER_ID,
'channel_id': CHANNEL_ID,
'message': 'Post message goes here',
'file_ids': [FILE_ID,],
'create_at': 0,
'pending_post_id': 'randomstuffogeshere',
}))
I have done a version for API v4, with the use of a personal access token. https://docs.mattermost.com/developer/personal-access-tokens.html
import os
import json
import requests
SERVER_URL = "YOUR_SERVER_URL"
CHANNEL_ID = "YOUR_CHANNEL_ID"
FILE_PATH = './test.jpg'
s = requests.Session()
s.headers.update({"Authorization": "Bearer YOUR_PERSONAL_ACCESS_TOKEN"})
form_data = {
"channel_id": ('', CHANNEL_ID),
"client_ids": ('', "id_for_the_file"),
"files": (os.path.basename(FILE_PATH), open(FILE_PATH, 'rb')),
}
r = s.post(SERVER_URL + '/api/v4/files', files=form_data)
FILE_ID = r.json()["file_infos"][0]["id"]
p = s.post(SERVER_URL + '/api/v4/posts', data=json.dumps({
"channel_id": CHANNEL_ID,
"message": "YOUR_MESSAGE",
"file_ids": [ FILE_ID ]
}))
EDIT:
I have created a simple CLI.
https://github.com/Tim-Schwalbe/python_mattermost
as per #George , you cann't sent the file to the incoming webhook directly.
below is code to send the file to the channel
from mattermostdriver import Driver
team_name = "<name of your team in mattermost>"
channel_name = "<channel name>" # name of channel which you want to upload document
file_path = "<file to uploaded >" # name of the file to upload
message = "<message to sent on channel>"
options = {
"url": "", # url of your mattermost acocunt https://<url>
"port": 8065, # port of the website
"password": "<account password>",
"login_id": "<login id>",
"token": None
}
x = Driver(options=options)
# loggin into the mattermost server
x.login()
# getting team id
team_id = x.teams.get_team_by_name(team_name)['id']
# getting channel id
channel_id = x.channels.get_channel_by_name(team_id, channel_name)['id'] # give channel id
#setting up the options
form_data = {
"channel_id": ('', channel_id),
"client_ids": ('', "id_for_the_file"),
"files": (file_path, open(file_path, 'rb'))
}
pp = x.files.upload_file(channel_id, form_data)
file_id = pp['file_infos'][0]['id']
# uploading the file
x.posts.create_post({'channel_id': channel_id, "message": message, "file_ids": [file_id]})
# logout from the server
x.logout()