So i got this html from a website:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:fb="http://www.facebook.com/2008/fbml" dir="rtl" lang="he" id="vbulletin_html">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta id="e_vb_meta_bburl" name="vb_meta_bburl" content="http://www.fxp.co.il" />
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<base href="//www.fxp.co.il" /><!--[if IE]></base><![endif]-->
<link rel="canonical" href="http://www.fxp.co.il/login.php?do=login" />
<link rel="shortcut icon" href="//images.fxp.co.il/images3/fav.png">
<meta name="generator" content="vBulletin 4.2.2" />
<meta name="keywords" content="FXP,פורום,פורומים,fxp,משחקים,סרטים,כיף,רשת,מחשבים,הורדות,הורדה,סרגל כלים,בדיקת IP,העלאת תמונות" />
<meta name="description" content="מחפשים אתר פורומים ? אתר FXP מכיל קהילות פורומים, משחקים, תמונות גולשים ועוד. הכנסו עכשיו אל קהילות הפורומים של FXP" />
<meta property="fb:app_id" content="415294715208536" />
<meta property="og:site_name" content="FXP" />
<meta property="og:description" content="מחפשים אתר פורומים ? אתר FXP מכיל קהילות פורומים, משחקים, תמונות גולשים ועוד. הכנסו עכשיו אל קהילות הפורומים של FXP" />
<meta property="og:url" content="http://www.fxp.co.il" />
<meta property="og:type" content="website" />
<link rel="stylesheet" type="text/css" href="//images.fxp.co.il/css_static_main/main_fxp_20.2.14.css?v=7.11" />
<link href="//www.fxp.co.il/clientscript/awesome/css/font-awesome.min.css" rel="stylesheet">
<script type="text/javascript" src="//images.fxp.co.il/clientscript/yui-2.9.js"></script>
<script type="text/javascript">
<!--
var SESSIONURL = "";
var SECURITYTOKEN = "1456672267-7067c7f37055c9dd77a4fa83ba3b7b6f316c82b1";
var IMGDIR_MISC = "//images.fxp.co.il/images_new/misc";
var IMGDIR_BUTTON = "//images.fxp.co.il/images_new/buttons";
var vb_disable_ajax = parseInt("0", 10);
var SIMPLEVERSION = "4116";
var BBURL = "http://www.fxp.co.il";
var LOGGEDIN = 1152224 > 0 ? true : false;
var THIS_SCRIPT = "login";
var RELPATH = "login.php?do=login";
var PATHS = {
forum : "",
cms : "",
blog : ""
};
var AJAXBASEURL = "http://www.fxp.co.il/";
//var AJAXBASEURL = "//www.fxp.co.il/";
// -->
</script>
<script type="text/javascript" src="//images.fxp.co.il/clientscript/vbulletin-core.js"></script>
<script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
<script type="text/javascript" src="//images.fxp.co.il/css_static_main/jquery.cookie_new.js"></script>
<script type="text/javascript">
$(document).ready(function(){
$("#ajax").load('notifc.php?userid=1152224');
$("#noti").click(function () {
$("#ajax").load('notifc.php?userid=1152224');
});
$("#ajax_likes").load('likesno.php?userid=1152224');
$("#noti_like").click(function () {
$("#ajax_likes").load('likesno.php?userid=1152224');
});
});
</script>
<script type="text/javascript" src="//images.fxp.co.il/clientscript/set.js?v=6.5"></script>
<script type="text/javascript" src="//images.fxp.co.il/clientscript/lazyload.js"></script>
<script type="text/javascript">
$(function() {
if (getCookie_bar('bbc_lazyload_fxp') != '1') {
$(".postbody img").lazyload({placeholder : "clear.gif", effect: "fadeIn"});
}
});
</script>
<script type="text/javascript" src="//images.fxp.co.il/advertising/ads.js"></script>
<script type="text/javascript" src="//images.fxp.co.il/skinfxp/s.php"></script>
<title>FXP</title>
<script type="text/javascript">
var forumname = "";
var fxpcategory = "none";
</script>
<script type='text/javascript'>DY = {scsec : 8765235 ,API: function(){(DY.API.actions = DY.API.actions || []).push(arguments)}};</script>
<script type='text/javascript' src='//dy2.ynet.co.il/scripts/8765235/api_static.js'></script>
<script type='text/javascript' src='//dy2.ynet.co.il/scripts/8765235/api_dynamic.js'></script>
<script type="text/javascript">
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-598971-1', 'auto');
ga('require', 'displayfeatures');
ga('send', 'pageview');
</script>
<script type="text/javascript">
ga('set', 'dimension1', 'Registered');
</script>
</head>
<body>
<div class="standard_error">
<form class="block vbform" method="post" action="http://www.fxp.co.il/" name="postvarform">
<h2 class="blockhead">מעביר...</h2>
<div class="blockbody formcontrols">
<p class="blockrow restore">התחברת בהצלחה, Copy_Pasta.</p>
</div>
<div class="blockfoot actionbuttons redirect_button">
<div class="group" id="redirect_button">
לחץ כאן אם הדפדפן אינו מעביר אותך אוטומטית
</div>
</div>
</form>
</div>
<noscript>
<meta http-equiv="Refresh" content="2; URL=http://www.fxp.co.il/" />
</noscript>
<script type="text/javascript">
<!--
function exec_refresh()
{
window.status = "מעביר..." + myvar;
myvar = myvar + " .";
var timerID = setTimeout("exec_refresh();", 100);
if (timeout > 0)
{
timeout -= 1;
}
else
{
clearTimeout(timerID);
window.status = "";
window.location = "http://www.fxp.co.il/";
}
}
var myvar = "";
var timeout = 20;
exec_refresh();
//-->
</script>
</body>
</html>
The page above is supposed to redirect you to another page.
I'm looking to click on the button that redirects you immediately if you press it.
It's this button if anyone's wondering
This is the relevant code :
def login_and_redirect():
#login into the site
usrnm=raw_input("Please enter your username: ")
pswrd=raw_input("Please enter your password: ")
print("Logging in, please stand by...")
driver.find_element_by_id("navbar_username").send_keys(usrnm)
driver.find_element_by_id("navbar_password_hint").click()
driver.find_element_by_id("navbar_password").send_keys(pswrd)
driver.find_element_by_id("navbar_password").submit()
#redirect to another page in the site after logging in
driver.get("http://www.fxp.co.il/forumdisplay.php?f=236")
for some weird reason the function is stuck and phantomJS isn't beign redirected to the URL I wanted it to. My guess is it's because of the redirection page, so i'm trying to get around it by clicking the button. i'd appreciate any help, thanks :)
You can try below xpaths to click on that re-direct button
//div[#id='redirect_button']/a
or
//a[#href='http://www.fxp.co.il/']
Of-course you can also use css selector here..
Thanks,
Murali
Related
I'm building a project and I'm using MongoDB and Node.js. I converted a csv file to a dict using the df.to_dict(orinet = tight). When I print from the database it prints out the same thing multiple times and doesn't even access the other data and I have no idea what the problem is when printing from index.ejs
files = ["Brandeis_ClipperShipGrant_2023 .xlsx" , "Google_ClipperShipGrant_2023 .xlsx"]
for file in files:
filepath = file
data_xls = pd.read_excel(filepath, index_col=None)
data_xls.to_csv('csvfile.csv')
client = pymongo.MongoClient(".......")
df = pd.read_csv("csvfile.csv")
data = df.to_dict(orient = "tight")
db = client["ClipperShipGrantData"]
db.ClipperShipGrantData.insert_one(data)
index.js
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>ClipperShip</title>
<link rel="stylesheet" type="text/css" href="/css/style.css">
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
</head>
<body>
<header>
<div class="container">
<div class="logo">
<img src="/html/Images/logo.png" alt="ClipperShip">
</div>
<div class="main-title">
<h1>Clipper Ship Foundation</h1>
</div>
<div class="nav">
<ul>
<li>Go To Main Page</li>
<li>About</li>
<!-- print all arrays in data array in console -->
<% for(var i = 0; i < moviesList["data"].length; i++) { %>
<p>
<%= moviesList["data"] %>
</p>
<% } %>
<%= moviesList["data"] %>
<%= console.log(moviesList["data"]) %>
</ul>
</div>
</div>
</header>
<section>
<div id="drop-zone">
<p>Drop files here</p>
<input type="file" id="file-input" multiple>
<p id="error-message"></p>
<ul id="file-list"></ul>
</div>
</section>
<footer>
</footer>
<!-- <script src="/frontend/javascript/dynamic.js"></script> -->
<script src="/javascript/dynamic.js"></script>
</body>
</html>
server.js
const express = require('express');
const mongoose = require('mongoose');
const app = express();
const ejs = require('ejs');
app.set('view engine', 'ejs');
app.use(express.static(__dirname + '/frontend'));
mongoose.connect(".....")
const db = mongoose.connection;
const collection = db.collection('ClipperShipGrantData');
console.log(collection);
const moviesSchema = new mongoose.Schema({
index: Array,
columns: Array,
data: Array,
index_names: Array,
column_names: Array
});
const Movie = mongoose.model("Movie", moviesSchema);
app.get('/', (req, res) => {
collection.findOne({}, function(err, doc) {
if (err) throw err;
res.render('index', {
moviesList: doc
})
});
})
app.get('/data', (req, res) => {
collection.findOne({}, function(err, doc) {
if (err) throw err;
res.render('data', {
moviesList: doc
})
});
})
app.listen(4000, function() {
console.log('Server is running on port 4000');
})
I'm using bottle to create a simple retirement calculator of sorts, but I'm having trouble with the template file's handling of python code. For example, I have this code
<%
import statistics, numpy
medianStockReturn = []
def stockReturn():
global medianStockReturn
yearStockReturn = numpy.random.normal(11.4, 19.7, 1000)
yearMedianStockReturn = statistics.median(yearStockReturn)
yearMedianStockReturn = yearMedianStockReturn / 100 + 1
medianStockReturn.append(yearMedianStockReturn)
stockReturn()
end
%>
<!DOCTYPE html>
<html lang = "en-us">
<head>
<title>Retirement Calculator</title>
<meta charset = "utf-8">
<link rel="stylesheet" type="text/css" href="../static/retirementStyle.css">
</head>
<body>
<h2> this is a test; your stock return is {{medianStockReturn}}</h2>
</body>
</html>
However, this code results in an output of: "this is a test; your stock return is []"
As written, the function should append medianStockReturn with a generated value, but it does not, and I am not exactly sure why.
You need to move the end to after the end of function definition above stockReturn()
<%
import statistics, numpy
medianStockReturn = []
def stockReturn():
global medianStockReturn
yearStockReturn = numpy.random.normal(11.4, 19.7, 1000)
yearMedianStockReturn = statistics.median(yearStockReturn)
yearMedianStockReturn = yearMedianStockReturn / 100 + 1
medianStockReturn.append(yearMedianStockReturn)
end
stockReturn()
%>
<!DOCTYPE html>
<html lang = "en-us">
<head>
<title>Retirement Calculator</title>
<meta charset = "utf-8">
<link rel="stylesheet" type="text/css" href="../static/retirementStyle.css">
</head>
<body>
<h2> this is a test; your stock return is {{medianStockReturn}}</h2>
</body>
</html>
driver.find_element_by_class_name("lnkClassInfo").click()
time.sleep(2)
element = driver.find_element_by_css_selector("#popup input[value='BOOK THIS CLASS NOW']")
driver.execute_script("arguments[0].click();", element)
ERROR: selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":"#popup input[value='BOOK THIS CLASS NOW']"}
Line 1 of my code allows me to click into the above time-slot on the Main Page, which triggers an Overlay to pop up. My goal is to click the class-booking button on the overlay. Based on my understanding, Python needs to switch to the overlay/iframe. Please advise on how to code this for my attempts have been unsuccessful so far.
Relevant Main Page HTML:
<script type="text/javascript">
$(document).ready(function () {
setPageScroll('dashboard');
$(".club-selections a").click(function () {
$("#ctl00_cphContents_ddlClub").val($(this).attr('rel'));
$("form:first").submit();
});
$(".schedule-selection a").click(function () {
$("#ctl00_cphContents_ddlSchedule").val($(this).attr('rel'));
$("form:first").submit();
});
$(".club-selections a[rel=" + $("#ctl00_cphContents_ddlClub").val() + "]").addClass('selected');
$(".schedule-selection a[rel=" + $("#ctl00_cphContents_ddlSchedule").val() + "]").addClass('selected');
$(".lnkClassInfo").click(function () {
$.colorbox({
href: $(this).attr("href"),
title: $(this).attr("title"),
transition: 'fade', iframe: true,
width: 566, height: 600,
fixed: true
});
return false;
});
var FIREFOX = /Firefox/i.test(navigator.userAgent);
if (FIREFOX) {
$('.tbl-wrapper .tbl-container').scroll(function () {
$('title').html($(this).scrollLeft());
$(this).find('a.lnkClassInfo').css('margin-right', $(this).scrollLeft() + "px");
});
}
});
</script>
<a href='popup/class-info.aspx?tcl_id=307632' title='Class Info.' style='line-height:74px; height:74px; top:413px;' class='lnkClassInfo'><span class='class-name'>ICE II</span><span>- Zaf -<br />11:30 - 12:30 PM<br /><img src='https://trueclassbooking.com.sg/userfiles/class-tags/cover.jpg' alt='Cover' title='Cover' style='height:16px;' /></span></a>
Overlay HTML:
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="theme-color" content="#592954" /><meta name="viewport" content="width=1024, user-scalable=yes" /><title>
CBSS System - Class Info.
</title>
<link href="https://trueclassbooking.com.sg/member/css/style.all.css?v=2.22.03" rel="stylesheet" type="text/css" />
<!--[if IE]>
<link href="https://trueclassbooking.com.sg/member/css/ie.css?v=2.22.03" rel="stylesheet" type="text/css" />
<![endif]-->
<!--[if IE 8]>
<link href="https://trueclassbooking.com.sg/member/css/ie8.css?v=2.22.03" rel="stylesheet" type="text/css" />
<![endif]-->
</head>
<body>
<form name="aspnetForm" method="post" action="class-info.aspx?tcl_id=307632" id="aspnetForm">
<div>
<input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
<input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
</div>
<script type="text/javascript">
//<![CDATA[
var theForm = document.forms['aspnetForm'];
if (!theForm) {
theForm = document.aspnetForm;
}
function __doPostBack(eventTarget, eventArgument) {
if (!theForm.onsubmit || (theForm.onsubmit() != false)) {
theForm.__EVENTTARGET.value = eventTarget;
theForm.__EVENTARGUMENT.value = eventArgument;
theForm.submit();
}
}
//]]>
</script>
<div>
<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="4B353318" />
<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="/wEWAgKm0IbODAKt8aSHB8YJdcVegLzMqZZLYYPUOKYp+jf1" />
</div>
<!-- MESSENGER -->
<div id="message-wrapper" class="popup-message-wrapper" title="Click to hide."></div>
<!-- CONTENTS -->
<div id="popup">
<div class="class-info">
<div class="header">
<div class="left al">
<div class="time">11:30 - 12:30 PM</div>
</div>
<div class="right ar" style='display:none;'>
<span style="color:#b91be0"></span> SLOTS
<span style="color:#999999;">|</span>
<span style="color:#cc0066"></span> SLOTS AVAILABLE
</div>
</div>
<div class="header">
<div class="left al">
<div class="class-name">ICE II - Zaf</div>
</div>
<div class="right ar">
<a id="ctl00_cphContents_btnBook" class="btn-gradient" href="javascript:__doPostBack('ctl00$cphContents$btnBook','')">BOOK THIS CLASS NOW</a>
</div>
</div>
<hr />
<div class="description">
<div class="header">
<div class="left">
ICE II
</div>
<div class="right">
<img src='https://trueclassbooking.com.sg/userfiles/class-tags/cover.jpg' alt='Cover' title='Cover' style='height:16px;' />
</div>
</div>
<p><span style="font-size:24px"><strong>Indoor Cycling Experience (I.C.E.) I</strong></span><br />
Highly recommended for those who want a solid foundation in bike set-up & cycling technique. A high energy workout, this indoor cycling workout is paced with light and music settings to create a vibrant atmosphere.<br />
<br />
Push your limits and get intense through warm-up, sprints, climbs and cool-down segments , with changing body positions, pedal speed and resistance.<br />
<br />
<span style="font-size:24px"><strong>Indoor Cycling Experience (I.C.E.) II</strong></span><br />
This programme has proven to be one of the most well-received fat-loss programmes thus far! Simple yet hyper-challenging!<br />
<br />
<span style="font-size:24px"><strong>Indoor Cycling Experience (I.C.E.) III</strong></span><br />
Expectations from participants as well as instructors alike are astronomical! Must have had regular training with Spin II to attempt this class.
</p>
</div>
</div>
</div>
</form>
<script type="text/javascript" src="https://trueclassbooking.com.sg/member/js/script.all.js?v=2.22.03"></script>
<!-- JAVSCRIPTS -->
<script>
(function (i, s, o, g, r, a, m) {
i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function () {
(i[r].q = i[r].q || []).push(arguments)
}, i[r].l = 1 * new Date(); a = s.createElement(o),
m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g;
m.parentNode.insertBefore(a, m)
})(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga');
ga('create', 'UA-45242383-1', 'trueclassbooking.com.sg');
ga('send', 'pageview');
</script>
<!-- EXCEPTIONS -->
</body>
</html>
You should not switch to overlay as it's just a simple modal window (div node). And also target element doesn't looks like an input, but a link
<a id="ctl00_cphContents_btnBook" class="btn-gradient" href="javascript:__doPostBack('ctl00$cphContents$btnBook','')">BOOK THIS CLASS NOW</a>
Try below to wait for link to appear and click it:
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver.find_element_by_class_name("lnkClassInfo").click()
wait(driver, 5).until(EC.element_to_be_clickable((By.LINK_TEXT, "BOOK THIS CLASS NOW"))).click()
Here you can find more info about what is iframe and how it might looks like...
I am dealing with a legal issue, and built a script so I didn't have to search a website by hand.
Script:
import sys, urllib
servno = 2000
servernomax = 2676
alldat = ""
while True:
newdat = ""
url = "http://coc-servers.com/servers/"+str(servno)
wp = str(urllib.urlopen(url).read())
print wp
ind1 = wp.find('"IP: "')
if ind1 != -1:
ind1 += 7
ind2 = wp.find('http',ind1)
ind3 = wp.find('"',ind2)
IPurl = wp[ind2:ind3]
newdat += IPurl
ind4 = wp.find("<th>Webiste</th>")
if ind4 != -1:
ind4 +=22
ind5 = wp.find('http',ind4)
ind6 = wp.find('"',ind5)
Website = wp[ind5:ind6]
newdat += ", "
newdat += Website
alldat += newdat
servno +=1
#print ind1, ind4
if servno > 2676: break
print alldat
sys.exit()
Bug free, however some values need tweaking.
The output?
<!DOCTYPE html>
<!--[if lt IE 7]> <html class="no-js ie6 oldie" lang="en-US"> <![endif]-->
<!--[if IE 7]> <html class="no-js ie7 oldie" lang="en-US"> <![endif]-->
<!--[if IE 8]> <html class="no-js ie8 oldie" lang="en-US"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en-US"> <!--<![endif]-->
<head>
<title>Access denied | coc-servers.com used CloudFlare to restrict access</title>
<meta charset="UTF-8" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" />
<meta name="robots" content="noindex, nofollow" />
<meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1" />
<link rel="stylesheet" id="cf_styles-css" href="/cdn-cgi/styles/cf.errors.css" type="text/css" media="screen,projection" />
<!--[if lt IE 9]><link rel="stylesheet" id='cf_styles-ie-css' href="/cdn-cgi/styles/cf.errors.ie.css" type="text/css" media="screen,projection" /><![endif]-->
<style type="text/css">body{margin:0;padding:0}</style>
<!--[if lte IE 9]><script type="text/javascript" src="/cdn-cgi/scripts/jquery.min.js"></script><![endif]-->
<!--[if gte IE 10]><!--><script type="text/javascript" src="/cdn-cgi/scripts/zepto.min.js"></script><!--<![endif]-->
<script type="text/javascript" src="/cdn-cgi/scripts/cf.common.js"></script>
</head>
<body>
<div id="cf-wrapper">
<div class="cf-alert cf-alert-error cf-cookie-error" id="cookie-alert" data-translate="enable_cookies">Please enable cookies.</div>
<div id="cf-error-details" class="cf-error-details-wrapper">
<div class="cf-wrapper cf-header cf-error-overview">
<h1>
<span class="cf-error-type" data-translate="error">Error</span>
<span class="cf-error-code">1010</span>
<small class="heading-ray-id">Ray ID: 24730841e07509a6 • 2015-11-18 10:36:04 UTC</small>
</h1>
<h2 class="cf-subheadline" data-translate="error_desc">Access denied</h2>
</div><!-- /.header -->
<section></section><!-- spacer -->
<div class="cf-section cf-wrapper">
<div class="cf-columns two">
<div class="cf-column">
<h2 data-translate="what_happened">What happened?</h2>
<p>The owner of this website (coc-servers.com) has banned your access based on your browser's signature (24730841e07509a6-ua48).</p>
</div>
</div>
</div><!-- /.section -->
<div class="cf-error-footer cf-wrapper">
<p>
<span class="cf-footer-item">CloudFlare Ray ID: <strong>24730841e07509a6</strong></span>
<span class="cf-footer-separator">•</span>
<span class="cf-footer-item"><span data-translate="your_ip">Your IP</span>: 64.18.227.167</span>
<span class="cf-footer-separator">•</span>
<span class="cf-footer-item"><span data-translate="performance_security_by">Performance & security by</span> <a data-orig-proto="https" data-orig-ref="www.cloudflare.com/5xx-error-landing?utm_source=error_footer" id="brand_link" target="_blank">CloudFlare</a></span>
</p>
</div><!-- /.error-footer -->
</div><!-- /#cf-error-details -->
</div><!-- /#cf-wrapper -->
<script type="text/javascript">
window._cf_translation = {};
</script>
</body>
</html>
Alright, so it wor- wait.. What? Access Denied? I have been banned? Based on my browser?
How can I get around this? I'm aware CloudFlare was built to prevent DDoSing, but, this is not a DDoS at all.
I would try implementing a delay, however, the first through last response is the same message.
Would implementing multiple browser agents and a delay fix it, or am I done for?
Following the docs over at http://wolfprojects.altervista.org/articles/change-urllib-user-agent/ , I was successfully able to run the script without error, or cloudflare banning me.
The new script is:
import sys
from urllib import FancyURLopener
servno = 2224 #2000
servernomax = 2676
alldat = ""
class MyOpener(FancyURLopener):
version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11'
mopen = MyOpener()
while True:
newdat = ""
url = "http://coc-servers.com/servers/"+str(servno)
wp = str(mopen.open(url).read())#str(urlopen(url).read())
#print wp
ind1 = wp.find('IP: ')
if ind1 != -1:
ind1 += 7
ind2 = wp.find('http',ind1)
ind3 = wp.find('"',ind2)
IPurl = wp[ind2:ind3]
newdat += IPurl
ind4 = wp.find("<th>Website</th>")
if ind4 != -1:
ind4 +=22
ind5 = wp.find('http',ind4)
ind6 = wp.find('"',ind5)
Website = wp[ind5:ind6]
newdat += ", "
newdat += Website
newdat += ";;; "
alldat += newdat
servno +=1
#print ind1, ind4
if servno > 2676: break
print alldat
sys.exit()
Who knew FancyURLOpener would be so useful? :)
The atom is:
<title>Example Feed</title>
<subtitle>A subtitle.</subtitle>
<link href="http://example.org/feed/" rel="self" />
<link href="http://example.org/" />
<id>urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6</id>
<updated>2003-12-13T18:30:02Z</updated>
<author>
<name>John Doe</name>
<email>johndoe#example.com</email>
</author>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03" />
<link rel="alternate" type="text/html" href="http://example.org/2003/12/13/atom03.html"/>
<link rel="edit" href="http://example.org/2003/12/13/atom03/edit"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
The GeoRSS is:
< georss:point>45.256 -71.92< /georss:point>
The google.maps.KmlLayer is:
var myLatlng = new google.maps.LatLng(49.496675,-102.65625);
var myOptions = {
zoom: 4,
center: myLatlng,
mapTypeId: google.maps.MapTypeId.ROADMAP
}
var map = new google.maps.Map(document.getElementById("map_canvas"), myOptions);
var georssLayer = new google.maps.KmlLayer('http://api.flickr.com/services/feeds/geo/?g=322338#N20〈=en-us&format=feed-georss');
georssLayer.setMap(map);
How can I create a useful GeoRSS that contains 1 and 2 for 3?