I have the following lambda function which is working fine to start the databricks cluster, when invoked. Now, I would like to add another lambda function and run it in sequence say after 60 seconds of interval. I tried it by listing both lambda functions one after the other, but only the last one was executed, and the job failed since the cluster was in TERMINATED state. Can someone please help me in running the job, after the cluster is STARTED.
Lambda for STARTING databricks cluster:
const https = require("https");
var tokenstr = "token:xxxxxxxxaaaaaabbbbbccccccc";
exports.handler = (event, context, callback) =>
{
var data = JSON.stringify({
"cluster_id": "2222-111000-123abcde"
});
var start_cluster_options = {
host: "aaa.cloud.databricks.com",
port: 443,
path: "/api/2.0/clusters/start",
method: "POST",
// authentication headers
headers: {
"Authorization": "Basic " + new Buffer(tokenstr).toString("base64"),
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(data)
}
};
var request = https.request(start_cluster_options, function(res){
var body = "";
res.on("data", function(data) {
body += data;
});
res.on("end", function() {
console.log(body);
});
res.on("error", function(e) {
console.log("Got error: " + e.message);
});
});
request.write(data);
request.end();
};
Function to run the databricks job from lambda:
exports.handler = (event, context, callback) => {
var data = JSON.stringify({
"job_id": 11111
});
var run_job_options = {
host: "aaa.cloud.databricks.com",
port: 443,
path: "/api/2.0/jobs/run-now",
method: "POST",
// authentication headers
headers: {
"Authorization": "Basic " + new Buffer(tokenstr).toString("base64"),
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(data)
}
};
var request = https.request(run_job_options, function(res){
var body = "";
res.on("data", function(data) {
body += data;
});
I would like to have both START / RUN_JOB in the same lambda functions, if its not the better approach, please help me, am new to LAMBDA invocations.
UPDATE:
I have modified my code as suggested by #Dudemullet, and getting an error message "2018-08-15T22:28:14.446Z 7dfe42ff-a0da-11e8-9e71-f77e93d8a2f8 Task timed out after 3.00 seconds", not sure, what am I doing wrong, please help.
const https = require("https");
var tokenstr = "token:xxxxxxxxaaaaaabbbbbccccccc";
var data = JSON.stringify({
"cluster_id": "2222-111000-123abcde"
});
var data2 = JSON.stringify({
"job_id": 11111
});
var start_cluster_options = {
host: "aaa.cloud.databricks.com",
port: 443,
path: "/api/2.0/clusters/start",
method: "POST",
// authentication headers
headers: {
"Authorization": "Basic " + new Buffer(tokenstr).toString("base64"),
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(data)
}
};
var run_job_options = {
host: "aaa.cloud.databricks.com",
port: 443,
path: "/api/2.0/jobs/run-now",
method: "POST",
// authentication headers
headers: {
"Authorization": "Basic " + new Buffer(tokenstr).toString("base64"),
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(data2)
}
};
exports.handler = (event, context, callback) =>
{
https.request(start_cluster_options, function(res){});
setTimeout(() => {
https.request(run_job_options, function(res){});
callback(); // notify lambda everything is complete
}, 60);
};
I do lambda functions in python, but this function, I am extending from a sample, so I'm not sure on node.js coding.
****** END OF UPDATE ******
Ideally I would like to have it within the AWS lambda, not going into AWS Step functions, etc.
Thanks
You can do this with AWS Step Functions. It is basically like a workflow.
At a high level, this is what you may want to do:
1) Run your lambda to start the cluster and return cluster id or something.
2) Check cluster status every 10 seconds.
3) If the cluster is up, execute `submit job` lambda function.
Lets say you have this abstracted down to two functions.
startServer and runJob
Your lambda will run until you call the callback or the execution time (TTL) has expired. So you could write code that looked like this.
exports.handler = (event, context, callback) => {
https.request(start_cluster_options, function (res) {
setTimeout(() => {
https.request(run_job_options, function (res) {
callback();
});
}, 60);
});
};
Another easy way of doing this is with SQS. Lambdas can now use SQS as an event source. So you could create a message in an SQS queue and set its visibility timeout to whatever time you need. Sqs visibility timeout
Related
i am working on a django application currently deployed on azure app service.i includes generation of report in docx. format. in some scenerios i takes upto 5-6 minutes. at first i encountered activity timeout after 70 sec which i solved by using applicationHost.xdt to configure activitytimeout and request timeout.
then another timeout happened after 230 sec almost. which according to my knowledge is due to load balancer. i understand it is not configurable in app service.. do i tried another technique which basically does is that it keeps pinging backend from client after every 180 sec. but the problem didn't go away. pasting detailed code below.
Thanks in advance
function pinger(stop)
{
if (status == true)
{
return;
}
else
{
$.ajax({
async : false,
url:"{% url 'DummyUrl' %}",
type: "GET",
data: {},
beforeSend:function()
{
console.log('starting dummy')
},
success:function(response){
console.log('success dummy')
console.log(response.data)
},
complete:function(){},
error:function (xhr, textStatus, thrownError){}
});
return;
}
}
function generate_report()
{
var status = false;
$.ajax({
async : true,
url:"{% url 'GenerateReport' %}",
headers: { "X-CSRFToken": '{{csrf_token}}' },
type: "POST",
data: {},
beforeSend:function()
{
console.log('starting');
console.log('readonly');
},
success:function(response){
if (response.data.toString().includes('.docx'))
{
console.log(response);
var url_mask = "{% url 'download report' filename=12345 %}".replace(/12345/, response.data.toString());
console.log('document generated successfully here is the link');
status = true;
show_download_link(url_mask);
}
else{ console.log(response)}
},
complete:function(){
console.log('completed')
},
error:function (xhr, textStatus, thrownError){
Error_Link();
console.log('error+')
}
});
// pinger(stop)
setTimeout(function(){ pinger(status); }, 180000);
// setInterval(pinger(stop) , 120000);
}
$("#generate_report_form").on('submit', function(event){
event.preventDefault();
generate_report();
});
the code above calls two APIs simultaneously one for report generation and one for keeping connection alive. but timeout still happens. Please help....
In my React front end, I call Axios with post method successfully, in my Python Falcon backend parameters are received successfully and token is generated back,
problem is the code in .then or even .catch are never called, here is my front end code:
async submit() {
//var aluser = this.this.state.username;
await axios({
method: "post",
url: "http://127.0.0.1:8000/Login",
headers: {
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"
},
params: {
username: this.state.username,
password: this.state.password
}
})
.catch(error => {
console.log(
"here is the error on a post request from the python server ",
error
);
})
.then(res => {
console.log(res);
sessionStorage.setItem("token", res.data[0]);
});
}
Note: the order of .then .catch was switched before, same result.
Thanks in advance
try to use try/catch
const params = new URLSearchParams();
params.append('username', this.state.username);
params.append('password', this.state.password);
async submit() {
//var aluser = this.this.state.username;
try {
const res = await axios({
method: "post",
url: "http://127.0.0.1:8000/Login",
headers: {
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"
},
params
})
sessionStorage.setItem("token", res.data[0]);
} catch (err) {
console.log(
"here is the error on a post request from the python server ",
error
);
}
}
If your backend service is only returning a 200 response, axios will not call "then" because you haven't send response data back. I just sent an "OK" response payload back with my 200 status code. "then" was called as expected.
I am trying to post some json to a python flask server and am getting the following error:
OSError: Invalid chunk header
header params
let apiParams = {
host: "0.0.0.0",
port: "5000",
path: "/",
method: "POST",
headers: {
"Content-Type": "application/json"
}
};
the post request:
generatePostRequest(apiParams) {
let req = http.request(apiParams, function (res) {
console.log('Status: ' + res.statusCode);
console.log('Headers: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (body) {
console.log('Body: ' + body);
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
});
return req;
}
let req = this.generatePostRequest(apiParams);
req.write(JSON.stringify({text:"this is only a test"}));
console.log output
Headers: {"content-type":"application/json","content-length":"37","server":"Werkzeug/0.14.1 Python/3.7.0","date":"Fri, 12 Oct 2018 17:46:23 GMT"}
Body: {"message": "Internal Server Error"}
simple get request works
getRequest() {
let res = fetch('http://0.0.0.0:5000')
.then((response) => {
return response.json();
})
.then(function(data){
console.log(data);
return data;
})
.catch(function(e) {
console.log(e);
});
return res;
}
UPDATE
per suggestion in comments below (thank you #robertklep) I updated the following:
let req = this.generatePostRequest(apiParams);
req.write(json);
req.end();
It works now!
When you're using req.write(), Node.js will default to using "chunked transfer encoding", which means that each call to req.write() will send a chunk of data to the HTTP server, preceded by a bytecount.
My guess is that Werkzeug is timing out, because you're not ending the request (so Werkzeug is expecting a new chunk, or an end-of-request, but isn't getting it and at some point it throws an error).
To end the request, you need to explicitly call req.end() when you're done:
let req = this.generatePostRequest(apiParams);
req.write(JSON.stringify({text:"this is only a test"}));
req.end();
Or, if you have a fixed amount of data to send, you can combined req.write and req.end:
let req = this.generatePostRequest(apiParams);
req.end(JSON.stringify({text:"this is only a test"}));
I have a Node JS application. I want to send an image from the Node JS application to a REST API which is written in Python. The key and the inputs needed by the Python REST API are as follows
My problem is that I am able to POST a simple 'Hello World' string with the code I have written and get a response. However, when I try to send an image something goes wrong and I get no response.
var http = require('http');
var querystring = require('querystring');
// This is some dummy string data
var postData = querystring.stringify({
msg: 'hello world'
});
var fs = require('fs')
, path = require('path')
, certFile = path.resolve(__dirname, 'ssl/client.crt')
, keyFile = path.resolve(__dirname, 'ssl/client.key')
, caFile = path.resolve(__dirname, 'ssl/ca.cert.pem')
, request = require('request');
// I want to send an image from one server to another. What changes should I make to the options below for POSTing image data
var options = {
hostname: '127.0.0.1',
port: 8888,
path : '/predict',
//image: fs.createReadStream('car.jpg'), //Should I give something like this??
method: 'POST',
headers: {
'Content-Type': 'multipart/form-data',
'Content-Length': postData.length
}
};
var req = http.request(options, function (res) {
console.log('STATUS:', res.statusCode);
console.log('HEADERS:', JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY:', chunk);
});
res.on('end', function () {
console.log('No more data in response.');
});
});
req.on('error', function (e) {
console.log('Problem with request:', e.message);
});
req.write(postData);
req.end();
Please let me know what changes I have to make to this code to post an image.I read about the use of multer package. However, the examples that I came across were using JS on both ends. Since for me, I have a Python REST API , I cannot use that. PLease help since I have been struggling with it for some time now.
Edit 1: Based on #Jana's suggestion, I added the multipart within the options and tried, where image is the key and the value is fs.createReadStream('car.jpg') . However, at the python end, it does not get the 'image' key because of which I get a False response. What am I missing?
var options = {
hostname: '127.0.0.1',
port: 8888,
path : '/predict',
//image: fs.createReadStream('car.jpg'), //Should I give something like this??
multipart: [
{
'content-type': 'application/json',
body: JSON.stringify({'image': fs.createReadStream('car.jpg') })
}
],
method: 'POST',
headers: {
'Content-Type': 'multipart/form-data',
//'Content-Length': postImageData.length
}
};
Check this,
`request({
method: 'POST',
uri: 'http://127.0.0.1:8888/predict',
multipart: [{
'content-type': 'application/json',
body: JSON.stringify({
"key": "value"
})
},
{
body: fs.createReadStream('car.jpg')
}
],
},
function(error, response, body) {
if (error) {
return console.error('upload failed:', error);
}
console.log('Upload successful! Server responded with:', body);
})`
And also you can get the best examples from its own documentation Request - Simplified HTTP client
I'm dealing with the problem I've seen some people also struggling. I need some functionality based on Facebook like buttons events. So I catch the event of clicking button and on it call my function with ajax. But on each single click my function is called multiple times, varying from 3 to 8. Problem is that this function creates models that ought to be unique, and because of the instant multiple call - I get multiple objects.
How to prevent this ? I've tried setting global var, but without any luck :
def send_submission(request):
global BLOCKED
if BLOCKED == 0:
logging.debug("should be locked")
BLOCKED = 1
(... do something...)
BLOCKED = 0
html = render_page(request)
ajax = simplejson.dumps({
"html": html
}, cls=LazyEncoder)
return HttpResponse(ajax, mimetype='application/javascript')
And the js grabbing the call:
FB.Event.subscribe('edge.create', function(href, widget) {
$.ajax({
type: "POST",
url: "/submissions/add",
data: "href="+href+"&ip={{ IP_ADDRESS }}",
dataType: 'json',
success: function(data){
$("#submissions").html(data["html"])
}
});
return false;
});
I've tried locking it with js, but it does not work either.
<script type="text/javascript">
window.fbAsyncInit = function() {
var maxAjaxCallAllowed=1;
var openAjaxCalls = new Array();
function insertHtml(data){
openAjaxCalls.pop();
$(".list-submissions").html(data["html"])
}
FB.init({appId: '161771113844567', status: true, cookie: true, xfbml: true});
FB.Event.subscribe('edge.create', function(href, widget) {
alert("glos");
if(openAjaxCalls.length < maxAjaxCallAllowed){
openAjaxCalls.push(1);
$.ajax({
type: "POST",
url: "{% url register_vote %}",
data: "href="+href+"&ip={{ IP_ADDRESS }}",
dataType: 'json',
success: function(data){
insertHtml(data);
}
});
}
else{
alert('Server call not possible at this time');
}
});
};
(function() {
var e = document.createElement('script');
e.type = 'text/javascript';
e.src = document.location.protocol + '//connect.facebook.net/en_US/all.js';
e.async = true;
document.getElementById('fb-root').appendChild(e);
}());
</script>
If I am getting your issue correct, I think you should define a global variable in javascript. Say you have a global variable busy which should be set once a request is sent and unset when the request is completed.
Before every request you should check whether the variable is set and if its set do not send the request again.