I have a web scraping application written completely in python. The information that I am web scraping is behind a login and I am using Request.session to save login sessions. I am trying to port the code to Node.js and wasn't able to find anything similar to request.session on Node.js. Please let me know if such a thing exists. Thank you.
Your best bet is using axios, which allows for extensive customization of request headers and types of authentication.
You can add axios-cookiejar for cookie management.
Simple example:
const axios = require('axios').default;
const axiosCookieJarSupport = require('axios-cookiejar-support').default;
axiosCookieJarSupport(axios);
// you can add an Authorization header to all requests:
// axios.defaults.headers.common['Authorization'] = 'Bearer token';
const cookie = 'lang=en; token=hello';
const body = {message: 'I\'m a message body'};
axios.post(url, body, {
auth: {
username: 'username',
password: 'mypassword',
},
headers: {
'Cookie': cookie,
'Content-Type': 'application/json',
},
})
.then(res => console.log(res))
.catch(err => console.error(err));
Old Answer:
I believe the Request object of the default HTTP server does not have what you are looking for.
But if you opted to use the Express framework, you could find similar functionalities in the express-session package.
To ilustrate it's capabilities and simplicity, here's a slightly tweaked version of the code example provided in their documentation:
const express = require('express')
const parseurl = require('parseurl')
const session = require('express-session')
const app = express()
app.use(session({
secret: 'keyboard cat',
resave: false,
saveUninitialized: true,
}));
app.use((req, res, next) => {
if (!req.session.views) {
req.session.views = {}
}
// get the url pathname
const pathname = parseurl(req).pathname;
// count the views
req.session.views[pathname] = (req.session.views[pathname] || 0) + 1;
next();
});
app.get('/foo', (req, res, next) => {
res.send('you viewed this page ' + req.session.views['/foo'] + ' times');
});
app.get('/bar', (req, res, next) => {
res.send('you viewed this page ' + req.session.views['/bar'] + ' times');
});
app.use((req, res, next) => {
if (res.headersSent) {
return next();
} else {
console.error('Not found');
const err = new Error('Not Found');
err.status = 404;
return next(err);
}
});
app.use((err, req, res, next) => {
console.error(req.app.get('env') === 'production' ? err.message : err.stack);
if (res.headersSent) {
return next(err);
}
res.status(err.status || 500);
res.send(`${err.status} - ${err.message}`);
});
const http = require('http');
const server = http.createServer(app).listen(3000);
server.on('error', (e) => {
console.log('>>> Server error');
console.error(e);
});
console.log(`>>> Server running on localhost:3000`);
Related
Hi I’m figuring out how to share Plex library in dart.
I'm helping myself with this working script in python ( it works)
https://gist.github.com/JonnyWong16/f8139216e2748cb367558070c1448636
unfortunately my code returns an http error 400, bad request
note: When I run the dart code there are no shared library.
maybe the payload is not correct :|
Thank for any help
import 'package:http/http.dart' as http;
class Server {
String token, ip, port;
Server(this.ip, this.port, this.token);
share() async {
var server_id = '00000000000000000000000000000000'; fake id
var library_section_ids = '97430074'; // right id , it works in python
var invited_id = '50819899'; // right id , it works in python
var headers = {
'Accept': 'application/json',
'X-Plex-Token': token,
};
var data =
'["server_id": $server_id,'
' "shared_server":["library_section_ids":[$library_section_ids],'
' "invited_id":$invited_id]';
var res = await http.post(
Uri.parse(
'https://plex.tv/api/servers/$server_id/shared_servers/'),
headers: headers,
body: data);
if (res.statusCode != 200) {
throw Exception('http.post error: statusCode= ${res.statusCode}');
}
print(res.body);
}
}
void main(List<String> arguments) async {
var srv = Server('xxx.yyy.xxx.yyy', '32400', '000000000-1111');
await srv.share();
}
The old code has a few bug and don't encode perfectly the data
I solved with 'dio' package and this link helped me a lot https://reqbin.com/
If the user has no shared library this code add a new one
import 'package:dio/dio.dart';
class Server {
String token;
Server(this.token);
test() async {
var serverId = '';
var librarySectionIds = ;
var invitedId = ;
var dio = Dio();
var data = {
"server_id": serverId,
"shared_server": {
"library_section_ids": librarySectionIds,
"invited_id": invitedId
}
};
dio.options.headers['Accept'] = 'application/json';
dio.options.headers["authorization"] = "Bearer $token";
var response = await dio.post(
'https://plex.tv/api/servers/$serverId/shared_servers/',
data: data);
print(response);
}
I want the following code to be translated into GAS from python. I wrote the GAS version pasted below but it is not working. It must be something simple but I don't know the reason why I get this error. Any advice will be appreciated. Thanks.
import requests
requestId = "*******************"
url = "http://myapi/internal/ocr/"+requestid+"/ng"
payload={}
headers = {
'X-Authorization': 'abcdefghijklmn'
}
response = requests.request("POST", url, headers=headers, data=payload)
print(response.text)
I wrote this at the moment but I get bad request error.
function sending(yesorno, requestId) {
var requestId = "*******************"
var STAGING_KEY = "abcdefghijklmn"
var url = url = "http://myapi/internal/ocr/"+requestId+"/ng"
var data = {}
var options = {
'muteHttpExceptions': true,
'method': 'post',
'payload': JSON.stringify(data),
'headers': {
'X-Authorization': STAGING_KEY
}
};
//Error processing
try {
var response = JSON.parse(UrlFetchApp.fetch(url, options));
if (response && response["id"]) {
return 'sent';
} else {
//reportError("Invalid response: " + JSON.stringify(response));
//return 'error';
Logger.log('error')
}
} catch (e) {
//reportError(e.toString());
//return 'error';
Logger.log('error')
}
}
Modified Code
function sending() {
var requestId = "*************************"
var STAGING_KEY = "abcdefghijklmn"
var url = "http://myapi/internal/ocr/"+requestId+"/ng";
var data = {}
var options = {
'muteHttpExceptions': true,
'method': 'post',
'payload': data,
'headers': {
'X-Authorization': STAGING_KEY
}
};
try {
var response = JSON.parse(UrlFetchApp.fetch(url, options).getContentText());
Logger.log(response)
if (response && response["id"]) {
return 'sent';
} else {
//reportError("Invalid response: " + JSON.stringify(response));
//return 'error';
Logger.log('error1')
}
} catch (e) {
//reportError(e.toString());
//return 'error';
Logger.log('error2: '+ e.toString())
}
}
Error
error2: Exception: Bad request:
I understood your situation as follows.
Your python script works fine.
You want to convert the python script to Google Apps Script.
When your Google Apps Script is run, an error Exception: Bad request: occurs.
In this case, how about the following modification? When response = requests.request("POST", url, headers=headers, data=payload) is used with payload={}, I think that at Google Apps Script, it's 'payload': {}.
Modified script:
function sending() {
var requestId = "*******************"
var STAGING_KEY = "abcdefghijklmn"
var url = "http://myapi/internal/ocr/" + requestId + "/ng"
var data = {}
var options = {
'muteHttpExceptions': true,
'method': 'post',
'payload': data,
'headers': {
'X-Authorization': STAGING_KEY
}
};
try {
var response = JSON.parse(UrlFetchApp.fetch(url, options).getContentText());
console.log(response)
if (response && response["id"]) {
return 'sent';
} else {
//reportError("Invalid response: " + JSON.stringify(response));
//return 'error';
Logger.log('error')
}
} catch (e) {
//reportError(e.toString());
//return 'error';
Logger.log('error')
}
}
Note:
By the above modification, the request of Google Apps Script is the same as that of the python script. But if an error occurs, please check the URL and your STAGING_KEY, again. And, please check whether the API you want to use can access from the Google side.
Reference:
fetch(url, params)
So I'm trying to uploaded larger files to my site, and due to Herokus limitations with the sizes, I'm trying to upload them directly to S3. Heroku provides great documentation on how to do so, seen here. I'm following along with the guide, and adjusting my views.py based on this Git. The problem is my signing request doesn't work when trying to post to S3. I get the error
The request signature we calculated does not match the signature you provided. Check your key and signing method.
So i am unsure of how my signature is wrong or if there is something wrong in my javascript. Any help would be appreciated.
My views.py
def sign_s3(request):
"""
https://devcenter.heroku.com/articles/s3-upload-python
"""
if request.user.is_authenticated():
user = request.user.id
AWS_ACCESS_KEY = AWS_ACCESS_KEY_ID
AWS_SECRET_KEY = AWS_SECRET_ACCESS_KEY
S3_BUCKET = AWS_STORAGE_BUCKET_NAME
object_name = urllib.parse.quote_plus(request.GET['file_name'])
mime_type = request.GET['file_type']
secondsPerDay = 24*60*60
expires = int(time.time()+secondsPerDay)
amz_headers = "x-amz-acl:public-read"
string_to_sign = "PUT\n\n%s\n%d\n%s\n/%s/%s" % (mime_type, expires, amz_headers, S3_BUCKET, object_name)
encodedSecretKey = AWS_SECRET_KEY.encode()
encodedString = string_to_sign.encode()
h = hmac.new(encodedSecretKey, encodedString, sha1)
hDigest = h.digest()
signature = base64.encodebytes(hDigest).strip()
print(signature)
signature = urllib.parse.quote_plus(signature)
print(signature)
url = 'https://%s.s3.amazonaws.com/media/user_%s/%s/' % (S3_BUCKET, user, object_name)
return JsonResponse({
'data': '%s?AWSAccessKeyId=%s&Expires=%s&Signature=%s' % (url, AWS_ACCESS_KEY, expires, signature),
'url': url,
})
My javascript:
function getSignedRequest(file){
var token = $('input[name=csrfmiddlewaretoken]').val();
var xhr = new XMLHttpRequest();
xhr.open("GET", "/dashboard/sign_s3/?file_name="+file.name+"&file_type="+file.type);
// xhr.setRequestHeader('X-CSRF-Token', token);
xhr.onreadystatechange = function(){
if(xhr.readyState === 4){
if(xhr.status === 200){
var response = JSON.parse(xhr.responseText);
uploadFile(file, response.data, response.url);
}
else{
alert("Could not get signed URL.");
}
}
};
xhr.send();
}
function uploadFile(file, s3Data, url){
var awskey = getParameterByName('AWSAccessKeyId', s3Data);
var expires = getParameterByName('Expires', s3Data);
var signature = getParameterByName('Signature', s3Data);
var xhr = new XMLHttpRequest();
var token = $('input[name=csrfmiddlewaretoken]').val();
xhr.open("POST", s3Data);
xhr.setRequestHeader('X-CSRF-Token', token);
var postData = new FormData();
postData.append('AWSAccessKeyId', awskey);
postData.append('Expires', expires);
postData.append('Signature', signature);
postData.append('file', file);
xhr.onreadystatechange = function() {
if(xhr.readyState === 4){
if(xhr.status === 200 || xhr.status === 204){
//do something
}
else{
alert("Could not upload file.");
}
}
};
xhr.send(postData);
}
I've got the following code in Python:
import requests
fileslist = [('file[]',('user_query.txt', open('user_query.txt', 'rb'), 'text/plain')),
('file[]',('wheatData.csv', open('wheatData.csv', 'rb'), 'text/csv')),]
r = requests.post('url',
files=fileslist)
And I'm trying to convert it to a node.JS version. So far I've got this:
var request = require('request');
var fs = require('fs');
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log(body);
}
});
var form = req.form();
form.append('wheatData.csv', fs.createReadStream('wheatData.csv'));
form.append('user_query.txt', fs.createReadStream('user_query.txt'));
What am I doing wrong?
This is how you do it using express and body-parser module to parse the post request and fetch the files you need .This is what goes in your node.js server.
Import all the modules :
var express = require("express");
var bodyParser = require('body-parser')
var app = express(); //init express app()
var util = require('util');
//APP CONFIGURATION >> Skip this if you dont want CORS
app.use(express.static('app')); // use this as resource directory
app.use(function (req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
next();
});
Configure the post Url :
//url => url/for/mypostrequest
app.post(url, done, function (req, res) {
//Handle the post request body here...
var filesUploaded = 0;
//check if files present
if (Object.keys(req.files).length === 0) {
console.log('no files uploaded');
} else {
console.log(req.files);
var files = req.files.file1;
//If multiple files store in array..
if (!util.isArray(req.files.file1)) {
files = [req.files.file1];
}
filesUploaded = files.length;
}
res.json({message: 'Finished! Uploaded ' + filesUploaded + ' files. Route is /files1'});
});
Make sure all the modules are installed and present as dependencies in package.json
CODE for making an api post call from node..
Include the http module first in your server .
var http = require('http');
var querystring = require('querystring');
var fs = require('fs');
Theninclude following code to make a post request from node server
var file1, file2;
//Read first File ...
fs.readFileSync('wheatData.csv', function (err, data) {
if (err) {
console.log('Error in file reading...');
}
file1 = data;
//Read second file....
fs.readFileSync('wheatData.csv', function (err, data) {
if (err) {
console.log('Error in file reading...');
}
file2 = data;
//Construct the post request data..
var postData = querystring.stringify({
'msg': 'Hello World!',
'file1': file1,
'file2': file2
});
var options = { //setup option for you request
hostname: 'www.path/to/api/',
port: 80,
path: '/upload',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': postData.length
}
};
var req = http.request(options, function (res) {
console.log('STATUS:' + res.statusCode);
console.log('HEADERS:' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
res.on('end', function () {
console.log('No more data in response.');
});
});
req.on('error', function (e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write(postData);
req.end();
});
});
Please note that code has not been tested on live server , you may need to make alteration as per your configuration.
Also you can use other libraries like request or needler..etc to make post calls from node server as suggested in this post.
Long story short.
Having some problems with some coding in python.
working on a program to spoof mac address and open new trial session on ap,
(this is for education only.)
the program works with ap that has free 1 hour pass.
the program works like this:
shuts off the wifi adapter
runs .bat script, which changes mac address to random mac address.
turns wifi back on.
opens browser and sends login through with new mac address.
repeats every 1 hour.
the problem:
I am able to shut dowm, change address and bring back up. but,
how do i get the new address in a var and pass it through a web browser to re auth myself.
and open a new session of the trial?
import urllib.parse
import urllib.request
import os
import time
def main():
wifidown()
changemac()
wifiup()
reauth()
time.sleep(3600)
def wifidown():
os.system("wmic path win32_networkadapter where index=7 call disable")
def changemac():
os.system("C:\Users\username\somewhere\theprogram\theprogram.bat")
def wifiup():
os.system("wmic path win32_networkadapter where index=7 call enable")
def reauth():
url = 'https://server.name.com/anothername/?client-mac=mymacaddress&ap-mac=addressofap&hash=somehasvaluethatpassesthrough'
values = {'plan' : 'trial',
'zip code' : '12345',
'e mail' : 'anemail#someemail' }
data = urllib.parse.urlencode(values)
data = data.encode('utf-8')
req = urllib.request.Request(url, data)
response = urllib.request.urlopen(req)
the_page = response.read()
while True:
main()
Thank you.
Update**
Yes it helps. I was trying to do this way.
macid = "01:02:03:04:05:06"
mac = "?client-mac=" + macid
apid = "01:02:03:04:05:06"
ap = "&ap-mac=" + apid
hashid = "8JTsXjOC7jcCppExAqbsBmFF4kNMePA6wgrMaMI6MLDUO6bZsc8tgQMfEfGY%2Bp9aLEdXfTZCapb%2B93DotAFLlQ%3D%3D"
thehash = "&hash=" + hashid
url = 'https://the.ap.server.website/' + mac + ap + thehash
def main():
wifidown()
changemac()
wifiup()
urlfunction()
time.sleep(3600)
def wifidown():
os.system("wmic path win32_networkadapter where index=7 call disable")
def changemac():
os.system("C:\Users\esc_fn\Desktop\macchanger\macchange.bat")
def urlfunction():
webbrowser.open_new(url)
def wifiup():
os.system("wmic path win32_networkadapter where index=7 call enable")
while True:
main()
the ap uses a script called purchase.js to get the form data and send it.
the code for that is.
var pdate = new Date();
var offerlink = basepath + 'terms/';
$(document).ready(function(){
// Hide Sponsored Access form
$('#complimentary_spn').hide();
$('#xsubad').hide();
$('a#inline').fancybox({
hideOnContentClick: false,
showCloseButton: false,
margin: 0,
padding: 0
});
$('a.iframe').fancybox({
hideOnContentClick: false,
showCloseButton: false,
scrolling: 'no',
margin: 0,
padding: 0,
height: 510
});
$('#triggerComplete').fancybox({
hideOnContentClick: false,
showCloseButton: false,
margin: 0,
padding: 0
});
$('#rateplanid').change(function(){
// Clear all errors
clear_all_errors('#messageBox');
var planid = $(this).val();
if (planid > 0)
{
$('#complimentary_spn').fadeOut('fast');
$('#xsubad').fadeOut('fast');
$('#paid').fadeIn('fast');
// Set offer and restrictions link
$('#offerlink').find('.pop').attr('href', offerlink+'ppu');
}
else
{
$('#paid').fadeOut('fast');
$('#complimentary_spn').fadeIn('fast');
if ($.inArray(planid, do_reg) < 0)
$('#xsubad').fadeIn('fast');
// Set offer and restrictions link
$('#offerlink').find('.pop').attr('href', offerlink+planid);
}
// Set plan cookie to expire in 10 minutes
pdate.setTime(pdate.getTime() + (10 * 60 * 1000));
setCookie('planid', planid, pdate, '/', '', '');
// Reset required fields
set_required_fields();
// Disable submit buttons
check_enable_submit();
$(this).blur();
});
// Set default plan
if (getCookie('planid'))
$('#rateplanid').val(getCookie('planid'));
else if (planid)
$('#rateplanid').val(planid);
// Trigger change to set required fields
$('#rateplanid').trigger('change');
$("#pwreset").click(function(){
$.post(
basepath + 'ajax/resetpw',
{
username: $('#resetuser').val()
},
function(data) {
if (data == '')
{
$.fancybox.close();
return;
}
set_error('resetuser', data);
}
);
});
$('input, select').not('#resetuser').change(function(){
$.post(
actionurl+'/validate',
$('#purchaseForm').serialize() + '&key=' + $(this).attr('name'),
function(data) { validate_done(data) }
);
});
$('input.submitOrderButton, input.startSessionButton').click(function(){
if ($(this).hasClass('opaque'))
return;
$.post(
actionurl+'/validate',
$('#purchaseForm').serialize(),
function(data) { validate_done(data) }
);
});
});
// Override validation error
validate_error = function(json_data)
{
//console.info('purchase.validate_error');
try
{
if (json_data.errors.nobilling)
{
// Pop payment form
$('.iframe').click()
return;
}
$.each(json_data.errors, function(key, msg) {
set_error(key, msg);
});
window.location.hash = '#messageBox';
}
catch (e)
{
console.error('purchase.validate_error - %s - %s', e.name, e.message);
}
};
// Override validation success
validate_success = function(json_data)
{
//console.info('purchase.validate_success');
try
{
var planid = $('#rateplanid').val();
// For Sponsored Access, perform login
if ($.inArray(planid, ['spn']) >= 0)
{
do_login();
return;
}
// For paid access, pop confirmation screen
$('#completePopup').html(json_data.data.pophtml);
$('#triggerComplete').click();
// Track with Omniture
var s = s_gi('comcastwificaptiveportalprod');
s.tl(this,'o','Payment Confirmation Desktop Page');
return;
}
catch (e)
{
console.error('purchase.validate_success - %s - %s', e.name, e.message);
}
};
var confirmed = function()
{
$.fancybox.close();
do_login();
};
var set_required_fields = function()
{
//console.info('purchase.set_required_fields');
// Clear required fields
$('.required').removeClass('required');
var planid = $('#rateplanid').val();
if (planid > 0)
{
// Set required fields
$('input#username, input#password, input#password1, input#password2').addClass('required');
$('input#firstname, input#lastname, input#email').addClass('required');
$('#paymentFormInputs1, #paymentFormInputs2').find(':input').each(function() {
$(this).not('#storeinfo').addClass('required');
});
}
else
{
// Set required fields
$('#complimentary_'+planid).find(':input').each(function() {
$(this).addClass('required');
});
}
};
My question is:
how do I get the new mac address in the variable called macid
how do I get the ap mac address in the variable apid
how do I get the hash it is asking for and put it in the variable hashid
how do I send the appropriate form data through. ie.. plan id.
You just need to auth yourself on the authentication server right? I think process of authentication and re-authentication is the same. Recommend using requests module. Here is link for you.
Try authentication code like this:
>>> from requests.auth import HTTPBasicAuth
>>> requests.get('https://api.github.com/user', auth=HTTPBasicAuth('user', 'pass'))
<Response [200]>
Hope it helps. :)