I'm trying to scrape this site https://acsd.crimegraphics.com, but I am getting only one record. Also, I don't know how to do this in a loop. I'm not able to scrape specific data. Here is the code that I have come up with so far.
import re
import requests
headers = {}
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
headers["Accept-Encoding"] = "gzip, deflate, br"
headers["Accept-Language"] = "en-GB,en-US;q=0.9,en;q=0.8"
headers["Cache-Control"] = "max-age=0"
headers["Connection"] = "keep-alive"
headers["Content-Length"] = "31688"
headers["Content-Type"] = "application/x-www-form-urlencoded"
headers["Cookie"] = "ASP.NET_SessionId=4z2ypzsqxnug03cf5ztitpfx; ExistingVisitor=ExistingVisitor=Y; __utma=114720600.1520129764.1565270251.1565270251.1565270251.1; __utmc=114720600; __utmz=114720600.1565270251.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; __utmb=114720600.1.10.1565270251"
headers["Host"] = "acsd.crimegraphics.com"
headers["Origin"] = "https://acsd.crimegraphics.com"
headers["Referer"] = "https://acsd.crimegraphics.com/2013/default.aspx"
headers["Upgrade-Insecure-Requests"] = "1"
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
payload = {}
payload["__EVENTTARGET"] = "MainMenu$InmatesMenu"
payload["__EVENTARGUMENT"] = "InmatesMenu"
url = "https://acsd.crimegraphics.com/2013/default.aspx"
r = requests.post(url, headers=headers, data=payload)
#print(r.text)
headers1 = {}
headers1["Accept"] = "*/*"
headers1["Accept-Encoding"] = "gzip, deflate, br"
headers1["Accept-Language"] = "en-GB,en-US;q=0.9,en;q=0.8"
headers1["Cache-Control"] = "no-cache"
headers1["Connection"] = "keep-alive"
headers1["Content-Length"] = "58838"
headers1["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8"
headers1["Cookie"] = "ASP.NET_SessionId=4z2ypzsqxnug03cf5ztitpfx; __utma=114720600.1520129764.1565270251.1565270251.1565270251.1; __utmc=114720600; __utmz=114720600.1565270251.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmb=114720600.2.10.1565270251"
headers1["Host"] = "acsd.crimegraphics.com"
headers1["Origin"] = "https://acsd.crimegraphics.com"
headers1["Referer"] = "https://acsd.crimegraphics.com/2013/default.aspx"
headers1["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
payload1 = {}
payload1["ScriptManager1"] = "UpdatePanel1|gvInmates_ob_gvInmatesPE"
payload1["__EVENTTARGET"] = "gvInmates_ob_gvInmatesPE 77886"
payload1["__EVENTARGUMENT"] = ""
payload1["__LASTFOCUS"] = ""
payload1["__VIEWSTATE"] = "ebMo2JN56yUxbK4AEzZhHzN+hv8txPcOkDCCBUqj0fuCztjMAgcrf6qhNyaJvycw4wNwvxA/ZGFR5frPHAWGBLOZ9WGTnV4Cj9KxGlGMJG8mJkdW+UKVf6GgCSU6OwHE4ehNxcz6hBdA0l44jmLbkD39FNNJabYM4HgcCyUkKq4XDDQy7v9/LkLUmndQ2h2SWx7n5mIVjfByM3DU2qH1G8mj5pt1D/eqRuv5vQR88mEP/B6XgMmXidkiJovfv2K2J8aWLsigBL/12HnRi9ZF4sn7wJx99WGxtB8Q6rz4CS3gOJYvWnNHtmENaeO8PURDe6HQ1CoOAy6MDhJplJzE2p6BAhBMRzbV0tRq0wEh/U/TBHY+3grRfmKZ5vLnAWqRdrgxi2jTr0o4Kt+HAtNwiJ2EZVl0RN8ZeVvPJBgtWFdn+K+hFUXjybyq6tTSwujPg/dgq27HlNFZbA4Y/0nvS/86FDmvXO/RXnP6/6qL4kigyUDIaAj8Q6N2i3Z3D3Wjz620RGvfVVp59OyJz12m2Ba+47s5KBKpmCMXiXRRjcYEz0v8yyp5bjWH4gZboEIuWais94XQlD0yWJ6lf5ioHC3aCbrTG3bwSWqoDqzdNmZLS+PrKz0Wm1tRehpF32id+v/+JWfM5NaYI6YuPqAgbieDrOBllCNHBdoSrYEnuACVEL+osLTp8g6mttuC+I+phKpZnj4kzGe9YxLnfd/m/cd22VJubXOiUjzlqGUOGq0yupj5wSJPuebQfNXwiWFzGAfRTbg8g33k3j/wHsawoYOGJQaggN2FhkcX00GK30UrlNi5VocnRV0F1zRZ62Q2u2C2AK6+SskMdZqT7oJv/kIRQcWSXRtSnjypHhuiW6v5dqiWMr2FoDAvYZh8P4+1vTsBztLZq8/bFRWzSnwDIz1wL8XvVQ49b8ATha2DuUqXJCimzgE5McBf/Ra8Hmea4Rc8GTiP043gBsIPzYQSGxDSyWFMjPrNIDfGsgoW/OMZ85YJKVf5KmM7j8vRBnXKgwDCK/pQZ78rvwm9kbqBG/OlZ44Ckez98/Ec1jrbcAj1kaBGEdp9Q2WCKHXo14q14qqqNxwcB0fEFEUoPcY3gj5s1GIBr1qQHGmtVnrBIMzt0+U2DfwTF9GCx9JsGnKNoEA2HxPKe2BsMqcYQohd6d1fHxC/0alMIjwxYnOE0lw1iCAN7PMv2iOMf3AWREVhV9qz+H9PRY0l1Nbd4io/BSPogFGxYVuz6RXOvOlA13xKtwbPjZVhDS0g4xXIGDD8Yr7HOD1Svl5Y1mB0hBH4bBlze1yxkqhlPNL2AidWCvbLwPDSmkVQFCJFFhcCj/vQxTxQZ+18pwe8lJfNKSlNjcoG+eTiD/JV3/nhCNrq5dnFsUCljohdwPviWIVuX2+Vxc15FxYLX79PhDg0hrOfZ9ODp96Cq2Qu+R01wnPlaiNvOhyFVhBkaWdsUgBXsRpDSW26srxPgn3gOmi5KdvhRen/kdJaYfu2cc94QiYLXCg1PvbH1JIGZtpA8g+fk3dvzIe3v4IRF3ODyaHYGVOSS636vPshxZ0wJLQRIqDmdHaS/VtJaD0lT7aDrOeaN2b6clVJvG/PltQdiZPY/ixG6Xm05ZpsfmkTltEASSuqe7NstquH6Of+TesFFamVujabDRvMxYAJ3vLeRmd1KG1Iuak4AqRel/ilRuQTW1BKv5w6UeIB1eUGZsPUVyM892nsMdt7DkVMijItW7nwLlDC+Gd7zhCod+ODn3sYFX11ejESCh4P/JncWTTejziJb6MFC/Ib19qQjpneO6pjmPV3yrhSsYL4zZUdvZd5LvWmWi68brb0rS9tRpW6oGCGsGaDuGMz9f3Iz2wBOhQoB26sDEy0+fwvHoFZAwNrRko5rBjzLESTys6yikOfhYLWV9SYML2YW3QrcEmED43JjcqcjAHgeiok3saEyPluEFe6Wq/HlSQEP+DuM+PSfkb+rxTBfjxm1+uZLSxGhOfLGtt4q19STBevFoPt5Zjca7Oqe21DKGQC7UNynqvKFAP8e2qbw4OHnxK4zAyNoMpCv5hPXxGWia/dDKsWPhmqGmyvyNoyTwEC85S7q6l2/iTww+2ENnNoKecAA3HCJ6yzbucsO1m37+gnK0hYYXNOzkx5AFS3GNZSJL8SOeRTH9v0xyBOxhYM57PSQjTf+PxPDUIiOfe4aw3UADWorUmZVlnff1jalNuODUs7XOBzZlXM73GR5IOE0tLJP7pr0Qlsf114ljlIpbRziTwhxUH4K8XOUBd7Vn64EJaHZJfu89dAQQEZQSa6heVuy1g/h5pJE6lPusTp5BmzUW7fLN+jJXi59R/wVH8WSjiPTqdVGbfThdhkLvgpbHBN8hZkCvis5GkRPwvEDhz7/WMeLi18aJUo3rqFHU1l8BHcaWjZVDnDsnOTS/xZ97rWZDgeu0OOnmowaks1SO4DxN6B0uVoX3dY2fLwBXGYv3LqTLMKL8QUZwnhtVikGOvd8KTd03e1sj9HSEbneZfpx6oQ44amCX2KeIZysJlrDooiBP+vUPzBSBqCWmjn/2/PyJ6g6srkI9v0ynkhaXXzGegGhr+ap1rh57SP6ng86W+ut43ucp2WFUAogvRBGwS7/CGmbT4px3caY5qI97Mlin+Qld1JQvOpPUPy25RciKZQwGb7AE7O3MOdp/bcTkOt0gc9Tw2aq47C793DvBphYTDkUy7GWWsUkzOudIS3mdzhR4ZoyBspyBw80R2gStjT26OjQlytD7DjDBuwLKUNcLUk2w8hjy9wbn1iqaOqtiFciqVxIu5mXSS31yy8C60lRCg+mEbVxUWlQswC//CfvaxA06MdW3UfrO//q3MJZCyfSdidKqed7AUqFS+ho/Aq4qz3+iw6rrEY5xLIRyetV4bHidxp7DroUwoy9jo2YfWtKksIRDHDM4VZm1iWkgykGZpvNlOAVAUqr/w/26j5orPdKl9vk0wiW0Wq3hCSO99TExEk8n+F3Jqc0Gtr3Q5uugb3u4G7X7uHN/uCJjUN1/6IpV5XllYFH5GNQiNk5jli0gX101r39fOXh40bBOKA5uClXgU43GldLk0ynNeTuf2o19Gfh+FtMYzwn7lGzYQoWkuiUmXCh/vvYW2yAOChpbaL+i+SmhmYNf0KtKdt58DKYMPMn89XcKKreN6lW3taArrp24eZGJKOUNbDlQOnMmNSlxLJclGtan0VZTTlDlRC17FzKE4sJ5mlKdcSa4p+uIDOeWT4Y8UvuXLk/gO9MMOO+XsYEMMjky5n9oQ9oexNc1hB/K3pc6iAZ7A46IRgaj9v6V4GrQgaB3Hn+eSSWOnrm7BuwZhtNVAIVyAixZuFMAyZxE4sexam7o8ku1DsFxcnTy8EyIb5InXjnOv3ZXNFB0b7gYBYc2ABg/H0PxFf9Fm83RejITIKoeeCiEbd4QWjCvjacLQGaZd8baDMjQeujl+t4c6UrydmwLDngcuDS3hUdkTNhsu+cG8B4xnBVA9ojBFtkcjwmdBz4BMBFPFZ2lh8W7fLpJ6+GV1Ls4NKOPQ0gCzTxQdK1ziYtZgEpVlRjlcpXZErwIKLD5ZQRcYj34vgTJ8o3Pf91Mp/SJ+cn8q9e0PKUqY3EbWWI8lOIsK/kVwPKhu9W3iEFnfWJ0HIKeq2YQwe2vBZyH3fBz9qICrWzk2vB7F+f0YECRsiS2QRfYUJ+RV0zMIuQDGr4M9TZry7Vh+fO5v0n9vHd6i7aexBMastrqFDRFr87HICLtrsBJ6Lq0TALoxgozihYZ3wDGlZkd8TLcW1OnABM9OX3xS89o4U+KyhOUW+RoTpFeNLvFL/3j0pPaosV+DIF3JDx9GHbe4T7sG5DoSizsbMOSgIR6P7N2AYmxTmdpBrgEvq6NZcFzPVRZko8CtFjWEMdrMqTnK7M5CAKLeiUIutlNaWcG25jw3RxqgVMpv7b48eVS5Te65S1+t0csMq4ja1/fqO55D0LTC4gJGwBI+2UIMcjvGQxOpEJHorKvKnXeC2VFrW89nDNvbvBiQH+gkVCV+VG4D+hActmA+eAWD6isrEdqKS+5k7GMkcDstYNi6yLyhaV7HF+Gp9w865PB0RYHyiE5lTWq1tNClgBHDmjo0xqMnr2hvKPwqE/tvtlqD1+b0mgRML5gYqVjJDPKR53mOiKDUyLyE74ZC39wJy27YljeJYzWp7l+keBGwDttQm6H24X6RfezFfF/lk8EkJ7PkUn5NLlF7LjDg/k+odOUMYow7rGfQ2eHNYlSMKT5uEEAsa9h3oK9SoRnbdjI9gOKw/3r523P+4Jq6Uu3aZnbf6AZNkE45oym5Pui2r907jekYSGy6vIRqZWeX3mFM0LfI3gitvr3hIdIZMB8Pom3I07Z61NdfWnEUGuIQiIL8OqjOHx8+J7d2cQSFdaWEJU2giriMHaYYJ1keNXvBGqJP8uPhbt1ZugBCmzbJLzREjQ9U+Jg7LvqOm6RJI1Eny0fVjWpoFyfjH1Bme6ps+KbxVptQMFqwKX7NqaPC2FtzwN1k/lS0ksJ05MAtyKFw+VLxHjCJqHkS3upoQjsukjwVHIqW3N41CubTvDU4q4ovYLXSyUVkfKrrVmysJ9zSqmpCH8HMZ9DZqDk3pwNYJmhPV7l7CIgHBNtalNQUNf68LmG4tqRz36VfFC3FZGEMWOaTRXVyExas0gfNd/wuRjLJfyLvyXPrhT6EfNKdgfI/W4wQ2Ooozts6GKoGu657pdJf7/mVxMM4d3cme7A9HduRIeFuqwf0Iun7tmV9E0WQBS9+pO1htq4L6s7twfzHRrmgA5ER8uLxbL6zN7ugrqUs1QGt4au3NnHVKxL86fuQ8s+RQsclrBLXrBMcqaBru4iHWg9LWbZvzPsNvAyRCxqBSIpa2RTw1sm+YFH3RH8gsIAVsXui91hZ4rb73GgdwQ0dvdyQ+bQRFDAt7eK9sTt6Jc0PR+W+O0DW2IErVisK0fsno4rLYyAbb4e28pHvAXJfHKcoP7udIybxAYUp3P0LDiKrTJIrrvCN9vTE8A8XpCeZuCuBwlV5b1wEVtUGk+h+ZNbfgbNuAQL6rLqbCsuGdhrlKiqUPZp/QbFskQkAQsPJmU6wIPU4MdZaqOINd1GwP9BJDhIpWUIMIwuMMW+OagZ73K6/yR0Xxv4AslGOCArGBRLhVlT9oNwhBJh48MyNso10j8hVSNfnnUoHkWkD0D99V3EQ+Nn97t6/O/qODME9IOpM87FSf5JavFCCyxuLRap1ZhKTBQ27xnJxpjy4UP4VR2K4xQjdqDSsCiyPiiO4ecU7jvHj27IO9mjJWsFH3RcySAp/w1bTicaFsUO9rlpRgtr1vr8ZQzfaRgL3JQXcn/ZRgFTWF0YMtmabG7joGhnqDKZhTF7PVUPR13EM12ZdRhPTd1/UNn91xlq2dtgXlAPoBAl9uoGwMdwBa7ACoSv/PZXi5r7lpk3a8rp1IsO7f2ovDtCz8vRf4fdoi21P48sYUif2Exo3u8YzAY/eYXfcq0/uF1v6S6jybYII3mtKqAVCZb38I0Xe3plOPa04aF5RHzqd7skZEnRragOWKYd8KzJKAhhERI4GtXtKUeyhHimPSQe6a2pn08zgnDggBPUEL2Z1D8i3qcJ7KGh5IdmauTHIblieX5zZeTCfb16CwepSLMktz1Z+573O66u62ttWlxTm6/NHLjpSono6s0ABoM/bie+xBjKjOvflwrwzV1v2wOnLJiEY4C0O7sKL4A/VRMY2ASf3fUI6nnJ97cxWWE3r4TbHOKVPhIyjQ5fx13UD0MyIb+5crqzzLz/x2NEPqdn0OBtu4WK1zwfnLT7Cd8heUEaZL2o2tZg7RQVei+bgs7bSZB84KUJZsrJr3OvtvxWaMEzREodJ1fyjTF+VR/ArXAlmU2418Cm0VVFAu1DF1zUry0uAlKQKiia76M6Dt1SdXzZEWYkz4Sv9cPQ07d7VNxa52MfzCOqf24SrMX/pKc2FK+McYic8Npuss2571WmDiR6P3A9bFoQtnT8BAU/aj+Y3OEecuCaRJKSokuyXvJTld4jbuUfefz1fh+o01ru0ScrKyOIw4MsXKUAi/lgXWbOesgmZ2PmBjX3/N9Vzw5IXFU3ZGxNZ43/BRwkBjdCl6pCsR5AggtSgguP0G4pbMKG8uA5+X7GwY2cOhM3AvF/ZoTiEdvpNZ+U3EgynBXDu1T0mS/Yv1WtGaObximIE3GkvG/GV6IK4CGKvTyrlnifrIN11AwN5zX3nApVXTQ0KCipMfUvf1nEeQpxAhM5PqK+hqx3KhJ5b5yEkMGlGWrWlf7LU1aZXXUQPOnEwwkQyhWpsHS4fee0uwDR8N3nLZ6QCfFKxNq6E08y9BJvP5qlap4L1ikGFjPN6kwYfL5cCOtwj99mOyO/EhegIlZTBsCAennCeDM0EXWvTwfaoTk0SLZK6KpKo8HD6jg8N6sXbBVJpfZNbOVdHzMpcpViqgF/XvxSxuhbhDLO1KO5R7TWOFQ70mG0wXiccktmzc1V+c+Af3Mz1xkh+KH+YA/Fn5xxDrYSo4K34A126a527wGoN/cBlNdsB0V5vWAGq93fGNUBD4IY3LnenrQDGaB6LhM02/yWsf+hLFwKOknn9za9S7iG+DP2OH0UGqwoWTLLETJ/RVwxC1VuxEuZlRd1XyjhNuhD5CKnk5lClPow+N8YLvNCytfRNuxe8DPzas2qomK/tptwN/C4A9uvrl7wIM4+AP3r9q1hNuKyxQNqV1V14QNcDYYWEqenjnUpzfHJZ2e2TfhLkSLN49YZ/fQVt5dUhvE1xOfePqs0iRJCkD/rMbdeYGZgoJIJuMgw4W4L57ML/bzzRWavGHRtvWk9WqBx3iWs4ZyELY0g5V4nQqFTGY38x2m0O9d7/PgVivq5R+iJKa8Y8XgGJ8l22uADue8665Kx5OYDF10DrvO5nFTYLoLXcQqmtSDO+bQLT5GJAOYzWDUcZGL5udJKuInN0PG2OrI7t/yszsdFihewuLMW12NfrOVTo+TAEGxl2IzJhSth+1jpBleawqfIgtZacvf7A5I29L6XFjXsM+X9QFqHuo/rPHhKUJnpRMyR4ywuVES3oTD8sEz7kyKTBl9qpu0OqD+c/1Tjc6YKWuwzpDLyiL0JeQm2Z3EpDu0Mac+hropbPIyN6UEOCqjKg8LALBxxnq6QahKdugt29h0QmXdxdEI46NyF+4FoEliyom6Vda4h+e1YGpn0+CfpUnY8F0skiJPTLizcH4r3bCuAWkJjYrbKNYEUFqyFsHpqyTi4MVr6zxxsPLSw7aWaOWlKH4k/Twiq+2THBRb7tKdcFnH2ujPyinA8s9C6w3eeZ/xg0scdWBCnJhYZ3wIJnigtdYr9rH5zmUKyec/Av2Eu4gARzPfxMdZAGi/E6d2wk5smLVewioWlAdciurHBtK3DabvNGVKx/R06HA6eTepQreJluMNEOrbuR1CL8SR7p6ZnKhimouVv99pCd0wWqo/3X+n56DxoAOFdwj3azxrdWcw3NaTWTLuf+MHPRApfZmzGQuF/vrqJhe+25b4+cgmELxEhglDsLu3H1JPQlVvU24bmmcsJEXdFWI4R32nb2Aj62LJxx88IUxIb3TGUeTPLWapo7aDWX7U11Eay/yqjysoJDt8/7vKM0BNDyHiFns0RU66w1t45whzONWVstXVwi3nK+LSzh8sVbFUVqCbnk5PpAkWz94phR7AMwHMjbjwNS5bp0Ev02gT6Cfwem1FIvQbqpLYzNgWl+FKOsOhlH0c7ThgNloVKUIoAuCyd36sRqVLH7JCKVwO2iHRJs7tp0pDRe8BLRbW/raFCNnPfOdxXvRl4fPyDZ7aIxdqMyM+nZ14lmlkFDca+y1d3GOLN1snUdz7l+TaAJwcb5IJ2aJZuxCEsyaUiqBXOr9X+1G/NxZQNqsDMdYrVQBMKk3lNpxc9/6IilypB+NbZ9C/HXKpcXt72FV29ZbzqNORHZbX7UJwb8D1wEN760+liGjkmqrtLof+VoIkslP9I/hIPrAkYhLo0VQlVhs3i2/uVsXMjCe/xZmLCWWA1XDx7afbP8OV4V1mgR6HinVE1hsxXlEE4J8EawoIrbUgmmvHr+plJQMqFQq27/5WqUhnyslUZO1LTi05g9eg/W3fsqF9Y2amVpUdpWGgxACrYWKewhzskMDkAscj+bx89yAhu7thjqMuhy9lsPmKa+HR2dwcHXdMdC3ffnrnhg3ax1gdPS3aXMZA8IncLn4F6s18PppI3h0cG2lMGgNtlpdvs7/oQ/kDOyeFVJNj9hTCi7gyRag4bIbkUnPVQYWXlBohcDHyHrVo/YU0ITI/nCBNl8ABQsvc2UtuNxRXMU3YKeaLIzKADW93gM9xL3VivHXo5vSLrkPFiHqGiAxV8dUpprN/GVUPOgepfHp4VghCRCQQvKCpuISlvj5cXIyUl7Gmb33RY4kQZC4YtudLW9k2Y0yZW+F+k7UCWYK3WUX2rruS3fk9VNaKGsSnfgodNHOSheBw1DIsHZHWNZpBVrUx4exoOtkHUUBbQf34tEIV0BSmhUBnSjJ6335QaUy+7Q9Pm1kSG0LeHTYNGJ0JQ2l9hd3THlYBKX9fhiG65xmygEBb0zK9L8T9UaIVMdyfeZdAlc9Wt/TC3opL9xS9c12S0QSlWsfRJ5zahKQoI1FdI+INfvP0O/yqLy/UAHHtV8K4SjQdNTrENdQjFJ2iCa/8EluE/6CwV4r7lL1zinMvlLPe0+y+4XNdshxAS5xU/BcQ3oCnKBNCkD49iqSECLObWvQNZTmGuUodsEfNCMUymA6I0Xzu+uAJrNKLL4XB/lraCfr7C6AOqgjAuJ0kGH4GrQBFXrHD73f7eH/0W8ekvJ+jZva/bXrMD/clmWXHilRfw/gqOpnZjf81hgYfWJtaxAz808+UQ1/Ur2hS5cXjxVrJyQOV4KTxFBBoC8TxFAD/QucvW0RkLZyaWAjILA5JdRHkSYH4tqHMG4jEaxvDDzATfloNvZJntcNZoKIUOQJPFpopo1azTA4hNFHazMub8eSVjxnVkyq0EjSrHzK8Px4Iy4vNxAUoumWYQQFh/RHzu65Ttd0v66zFrsv5j8pDmNGXIlUoXhD0Hz+0H2vEy+YT3Zi+aOZIRBh5pp9aDvhMver/MWYz5bmO5Xyf3s5orXMcgHMDrdrQgAlVkU8CwM5ZqghKqxslzbz/ntORVlRgR4SMqqBsOkZn8NWtIZUAw2UjXbW0rdVzwrnI7jPLQqqnm+wPiw9bR0Lz7lw5RxFawt8HR0THSF4YUwFReH5R3syEfVErZ+7U6bsDfKWr4vobI/DZufRlaVirY3qaXPibs+ESVCIxJ8OSn2mlW+dX3TBB4ydkJvs7m/7biCRBGdToL7W0JtQ8wTyLeMXGfsX88QrLnmxxOlwT3os2ZTwLJN1AjSuJG5Bo1IdJ1X64eqx6yI+XScU7HDRpUOBuSvQtAB4rHMw7GLnPjX7rOj93AIodhQRPpkAvTK7fRIHMZx0++wvvg8NmzrcObxecypL104WLLOb0YenSFZcqE5d7LwvX+gHKt1s+SwsJR2mS2ABCbrv9SNDH0y00T3Fmban4aLxx+F3p6/VRaFA89Qv9C8TJDMT0RfB6quWjn9kziVl7WpczqcT4czaGQkFGGYBssPpLYnSPBrHr5aSjXVHRAYwOWFYzJes/hNfI3NFDNeodIr9PeeD3jHq3djIK07lsN47crCctmaKmbVZvvIUaJw54UV0YG42v2OUMA1HY5Sa0h15MMT4cMHNIu8p3rw8LNfVh9SyfNWJIIcVJuiPYfurg+JuURZRMDyeq256i8tYu+03iUB9nbLCQegLsjldQteW45YvG5ttxJT6viNVo0Q/CEbxrcIhWBnXE3ciY9MAKSB13/zWg+OeI6Z75gwvJeuO1zVKSs+r+JCTWaJPBvJEHv1OTg7XC8I5XP6ZlTBbSU6H/cCmCW7hbcdI5YvpD53p12L4PxvGStDOFvlGOfqwA5q6NGnAr8rm8fp3V0wpXhek60O/HW6r2rUNP/OeR4CfCRFxFQhokdF7x8Z7nRt3tE/pbkz4GQ88FMrEPBsLmfr4odavit7aHajFALaDwuhtlpdPNS40Y/IPOG8zjv+yEiLD31MLlQfaDBDKCk7Vh3XGVRBbjlOSPipCONrwG0RddEf8pID1NTiJLrd1w1inomVC66tlZvcj1oTau3CA+bjEhM0pTO/ZyI6jCxDtTO4XpQdItJUz3j0J3CtZfprviPvXWMPEbqqRbvVc+gjFpYBPFrZ/aZoCOHtMrYIV6fnBqjrDIRTGuRZ85ZcvTac+8T5L//WMQuXmgenrcXbYwEDikWUntfSwM1Kp5EAoByzVvWU41y1Wty4hXnWjBxIxPQWdY/eXXm/+ixkTj0cogvvPW1u4R984yqZ4I7Sbcw6UMFW59pKNVt9ubmHaz7AV5BF6LrlosO2dAOGrXuzRT1eL1AevDUqum3usGSAz3hRzz6m9cG0xtba3IJhBkKGCWVCjf6/EtoBYpcAB7919RQX0sO2JE4R7HAy165x1BETYu57CCWqzG17Fb4fx6TYEuXFbSXtBC4et7ziPPk3rc4RkEyUA33zYGxO2KbSCH5Ilq+M8//OzZIh8lc7ESV0yqn3KUg+oIxcTIiDV5DBbfKs9BB/4NRc9nsP4+uL7lzlY5pcpbLvdLnW6H1RjAgZwaxDiMP8ML20O5qTdi424dBioIxEwTdSjY8xzBQIJk5Nhr5K5hvewBGOLbioVbLWz93yytLIoCzQ57MIPOuD1QEVlfe1KbGNKvx1UL/gl6TshCwpPlxuda7RBgSXTeIazMdlSJseFbyUFOnRbPEoF/4RZb3OE2gYyRpD7NQ1IZpPFWLldE4jZL1uNzwLXDhlzmzF4HFgEoEG6ZV2icn16YbMJMnd0UoEhXEMdyuwatMj5etTn7nGq6SPqxG7V3DNZWsTBiEIVnzrDCpG8rQcWU0Kq+HULavrzCsr0VH13sUftk7ZEXbPq8B+/Z2tAn0772vP8tlf7iazaQdcQGB8E4YZK6dDUYLoS0wJr4V8qP9H5kzkX2DhWG7awOIaihZcWZ6KblasF49PA19u4wGTehxg3E8SjiWfUcuI4EEiKOijfSdJDSEH67bnyt4SBY+a2HLOk4VZJGoHmz7KkBtgWMN9p2f5F20M9/QBzhn2K2ocIrNr+s2vm9NAVvKiTdmPAQIn3LT8Ku7rnL/gJkbPtYhtx+3pmB1r2Xnv9CsyjVZciqMmJhkleEZQadzsvLOL/kav7W1g93o3n3ywrdecks+4ss8wYDzS+TFw4+7ooFyX2DAlYFBGgy6dPAnzeLvH3tpLvAyvkSWSMR9q62tTlUbWHKeNfgBJTVV+x7VHWDTJTl84YjPx4Uly01cwJ/YvDlnxKqTLoD7RYNaSgypscVvD3hNwbd5tbsdDPAyeuvPh7kSI/sgSH1LQ3r+h4Bo6qMDxYQ9Vu8bjTwwsXs4nvpkH8n/uYBdi0FM/jsWqVDwKPEueoH7GJbVDAlAv/UmrWQLkSE3wczfN0vyKSJ81ogCFL9al43LkGl5Z9JsVYh/PtScawqG9wUTO6LKrMl/Z58uRCzTfTDlXYatj5X+HHs8BS9xqMNl1tW/+AFECms6fHAjxOgimpvppm40LwEcbWyvI8fVvWvQnDoXOYQxFEufVs08CMq7v+qoLv1m0MB+P1L99WmrWaCtpHP1HYf4PGGE4uV3ouXlogyrQwQSOJzhXKh8rlbHb+nt/FTLgz+KTWsvKtoaC7LgE4soyKoueB/jFVrGt+llT/GPErzdOKPzc4PwS3fh7RwjERvqGyVGNwBqtnT+3CBQ0X2Quqdpyju28OHeW1bt3BFiqE5PWNgpmQDmuUCD4j9yBIcRFEsiUHWfx1h+wcZOnfLSGTLZA2IzQUln+CHqs7y9mUpExKRnHZEPziV1DUZW+5rHDIQtzkW+KmJgu9SR08Trb7rC9Vpb74l+ar6S4G6fVsKUcp5uoFFIcKOm+RDaDl5djjjs5Q4zfonT2KGZ922vu8XjQx3GTo+A4ach2JR42eRL3FuLu0vLOK6CCcrdf4HDWSHH2j1JVcih+8UYURGDRu1Ya5qYFzmFyTCmcCBqVCah5AoL8+vPwqjWlzj2gxGUuLPDtWuotCJVyRRRHLFHJbVTBdTfhNYwjm9Pkj1rN4o5B44Jx6bM+y1RFWkzM2YamgIR7EtTb64UhYKXVOitEUPsgHOWxi1dAVK/grsvrXq/ImLB3z0MefUvjRZB/J35LzXTWKzX9Ydlrna4UwaKsZ0aWxBdIE2IRgG85hnjZEHK9H73TdFt+C1TdFOUkXh8F6YR4/HPQEbWwmu3jPhERFatByoaoYu/l2a/qZtSOpWIP01kXP9SU3ROVDmymPnIPo5PU5RzhoGgMgNOlSshpWGdtxsuORC7vfUl1ge2m7skxaesWeaCDHnXY0zLm9twiVRmYhdB4BI08Mdq6VqjdwijkRrMnKz1VXfwkVeP7YaQx6IL7+LSqx1dDHpRo/2JjDkM/KtlyKb4bxMJundZmaRrMkeg6WK0H43kpaXZIju+gRXbzP056Gpw/UjelRnL/B59ZTMUKzdfpJzzR4jOq2TXKccRWM/BVklMew9aU5TzLxA+zkXZ/T+xV5g4vD3SRgNp1e2reGOeLjeUCmC8az2aAfk0JyG/J09aIGs12j0yLUOqIv2HyqsE6ccfBthqgj9jf4hEX4XwfOQIGd6LGuUcRejZpMXqY+lulvfW1SI8BZxuRSr6hvNbRB7nbHWBgZB8JHD8sj/SXQdQkAglinUT+a9U4K6IwfMBYADwhQS5ytx6tsDtNdK7c4ZShDTa1+VEY5v5mg0pUUMPOZEN2X+SFpDX8UCOXtGC+JvrH2WWlbGr3/m2kIQKypYC745omqPtx7Jl2QFszs2uACLeyvVrjIMmHHKtEWvaeFAjtVEUzE9vX6ydveML9upl5gzXfTNmp8G83Isf7KX9OeNjdmuTOMOGWqXMs/cTSNfQRZ1MgPAh5mPGO/+FNTQWQlCbUGhQZNLzGYM5ujsiPi29tI9tmTGeD0YD/dOx4kvDC/D24xTx10Y8BYDeT1OwEtDDG0vZSdi3O8ePXcYvb+8qmk7bqehJUhv0lTI0CAOTqIKAD5MccNREMFrcgUOyWrFLzqDeI9YYcBQyaZRpStq4rj2JlvzRvBxyZEa99ikGTEHU0s7Oxv0dfCAx519E3D8pQQCUJnvkh2hCQc7XBWdLvW+rYkv/uUPHsVVx1V6blRnK72olRl39OM7rjmT9vB0lokGcHKswQ6KSD74NZL8/anDWHXigXZLRIsGIZaHkPgOR4jIvKfULYLV18u9gyeiVFwMhc16UfHLxhWgXj9bbAiFjRwcGqLpcJLRMtwCeCnGQHVnUVaE19fQ1wUYBz0N1c+7a4apqfSLyrgP0cAMhrFcs2h7GtSuOGlk+HR5xp/DRvbv5qP1UyhMuDjjqJHD9Q+8vKxVk2qcvhbiIFd9TRdwsfBzfai/aq0xBAH+BwGEoXSyA2EFl03XqI6OzpKREudSEGhUHVOX3YTYST3Hqs4lsOoVD4OHeIqwj7vSHmjftOB0+6skIu6tzG6Y7rAiYSS6o7JUVoXuSRNbS0uQpK7a/1jtcI+lW/TBfR+Mcieqivhgt2F0uigEadAdh17j6bOfxz8CNXjU3mQFaf1gGi/NS3SH8ZbEUNfr8Yd3vqZLxCFSJb30UcFYGTLd8aAZsvPUjUyxQqrnw3MoET2FMIZzZwysLyJhbXCKIGZ8NC4Vr1Md3LIpOr3YpLE009E6wTgQUFw0Ywx6QxWnDA+lA4xXESY63rgk4MX1mAR3Pa2dnnc1pEWs624pBudQMDzhGQoCM0pCUddufYuAem2lJsjRXmUnTdTHbt7Hj7MP3Dgtf2IUVG0lg00d4iVGOpOlbYo0R0GHrrED4/t+v1339E4QEmJiLiysaISFNGX9XBrSlrp4c9zDySH/GqO4FPM5/Niiexiv17CzutyYuwq0cgqL6GCE7jGz3vh1Ql5R9oeuIVQI0ZMLxZ3uHeQiglbgoV++pNbrY3LVtcGApe8REEdqc7fVkRYZDekWniqGuACEbpBO63Hsk1gYciImSaQC8Oy4+AlRGB3iWF3zVGpn0fEY1I83L/F6i4p+/ELLY/y6APuCls/+8idAi5J3KfmQP4+LmouwkRXufkHYSodaGWIBrPR4lxpDwIdZKDAvvIahMXje/1D6IUh9IpYMWlPDL0BvOknXvGcrmQjtov5WPlPrqxuhnDlzh/r8ZTqgkfJQUYdTvGqG8XcigzgLt/ZfbwbKfa09Azmp6IQWw9DgkH27BYhUxTZ10IUP+YaO1xIhpRd0KewajQSjsc5nxagZ5+vKmOfRzeUMnLqhMV64ns4deousPFBynAtxLQFt5oasgCOSi5SydqUMShV0WsIUJM6N62iHb1QLgCOAdxhbtiNA5i+Be13F+b3arqhye4yBAcSdaKDkzpDaDKgFJgMwfoTMfOCa74I5rRewAScs4/7u9p9r6VoP9jx87IFStgcluQq/djfHplH8HExNwWdZHzbBLwd8+0BSkQvTuR1krqBScgTZDVlSVYs8vn774d6dHPdwtN/7w9+SeKvU9ebvtxj5MaaoUuAqK4hQ/gqjer/QWuv72SuwAQAct6rI8gYwmNbela9z2g9gtGZ7h/ZFc4LD3D8R6tBDfxW93YoqV+EeeaGfNq+5/bI/kvxu67/uxiDDL//JNE+nHszOXA7jjWrCURCb8e0G8hRJgUX8ZvGJAXLQQY4bRogoC/zr/Wo7pLsw6ucSfX5I+xoS6Is/RSs6kwbRSoqvZtlj+sZzfXymF6x8ex0+fR/gvih8qqZhbh/p2C0H8e2vGHRh72r2aybjjbxFh4sdzSZYqoiYWLUFyek3Av1ejhcqf1e3QSTAThg43VlD9DkGP5my836D+NeK6rW6q1XEieSclxFVVMnZnOQ0Z1mWobTY9Oay2FvcIQeggcrKQ3jenG4ljgLhgpRwwIgMF3WMr9UteHqPJ0UL/v2IBygnk+kwE97Eo70xNg1pjgWpdNUUj6+veiez3EDRXG8//hpCwun8foWaZxi/dcb3uF2ntxwkDr6SLwY6GHebc3uzVz/PNhUpr6hvWvvUZPbOYSoP7KRQ09nCBM4lQUGVlabtt1VJAnJ5HfxDqOc+zxAcoqEHPFuZECzBrP4w0tQ+nJYbYcTlPd9dhGZq4NBMyHE9rFT+Tl5qlgkYpsDsrfUTrny/OW+cOepLR8Rpfyorltm/dgeb7b0b7PtkKiNDUdLemqLNMawJIER9dDtHBwXJYuDQfsZVOx3/0GFfkLVl0Qj0MJMXxkNqCvBtd0UUWAEH6JvlUxCCJ/wCKrwJ6tqb+nlM3Q4scUSA2q1Vm9//MXipcjRiBNeib9xywKSV5dJizyHNxPp/MYQyFN1Xy7Knkh7vTITWwoPIg3RWXsFW61uuzc5LZmT2nt1RxCInMPDdbxn99qTNNkobV/hUIAJSfxAS+pkIW7ma06fTKM/Sef98tCEzyy7aTVNjeBrC5hImA29o4l+kXXgklXbpOsNaER9imPHoxKdRjFD8DIkbrPpLy2kAklNcz+NduqCSv+/Y72FfhsEZ1N287PDcOJSOPRgsDmE3V9Hw4H2c9L1Tz79wU+VSY/oigWoOXyoOM7oSxYdhMfFiJEQg81E7E1bNQzYs+VJAOj+7idowNkgovgzsXIt4F2L7929aDb/0pqk9g67ZeHPQnZzp3WY3+Hw0guv/iTsJhNkXSIeZFEjlo8p9BPxDDpJK3PRKUYRvc43hQ5MF+Gg1fkHcHoy6gBO/YNkf+Y+kg4mCj3MS1eS20a1fMw8CCiS0k3M385ee+b1y+c4d2LQmRj0wrZ2yj5kcxekF/D5QrMU3v273iXP3991fyF/ao65JpVGJswok3zn4Wxmn9IKJdoEqyVw6WNeTwfiJqHLLsLTzKxrPru7vuW5XQfZnRFnniJK6QlGFVsXNCUCM8n8jk+BsAtGdJqxjtwu8srUgppZzxtfP1fFZL6mkVFtinnXnis/AduMX97iIhrcZHA44ingPR61uAiXS6uOwk/P99HCLt4COTPUMN+/WXGsmawDdOwbFqixArAcUh1rd2Ip9EGPoNRNaEXS9YD1+o/cZ1A+mxgiewXKM+1iCBlhV3RxZ3Jh15AHk8wmy0TRebza7SmMBg25G3HPfhJ3TNGFpWUhidjYB5vTKRM63P7sAcSFEX6SvOw1J0Qb/S8jAZZ3fQsLCpHzdEDdGXLrixkPV2W5SjbuhtI6VuSsukZMPtcoFBAQp3K/QQqbtvwM8HFokxhILDkj3F6K1knT0b1DKFSsRcnlCqPsx13mtKkR40uB9sRWpMHXPPrRvKZnwDd+s3ATbjBn4znPaQndcIKSrgT3PBhpkuAQy0e3s2NzTBvHUMo7N/70NN+te8ufZtztXYcTiI0vCnbmP7EJsG4dpnjmxHz/rVjUzXQWvZut9gXsvYH8uIvn7A6/l598PKOzlEdv58c9CkpJ/i4aL+/eWcaarAIdVnVTUXWiehziAwMdrBcviONNmH49t1cQPG3SbfPyBUFXyQA3sH1E1kQTGRP1rtdLMGWhb91PVFFVF8SQ0gnlk3kxvT/8gPXdc+tfBe6JXving1KE+hIPwHyzpANb2EQPID5Mv3jViRxQ9cm1QE57lRfMQZEsBmBk/yhquvYKapOoKfq4ZdspzjAgepCJz0EBIM6bZaTyWBGgGVJYYa0Ns0sukhahjBi9RPT0efJgwcZaTe1EWu/Kfo4nBVphREkDuH7jFbMDV+atXjlAkjhDZOgYBCmmFEykXAxfnnwfXqdYEdoe4otHRVpu7WDCDCO1G0joV43wwZoolDMk+3EA/e1HVuAnhOlHWon4FfWHP3N4ugHR84CMYkctT9CSthb7g32AUvI/EjogHM1rrmLRp7YwnghZEa2FVjZkxAlJISK8sM9ini0yKpCWz8wVTJB8+Mom35YJviLzHhEPamHydpC/tLar8Wy7lZ8S1qNswoomvM1+HzE/7WwU4NqvGeC7K9Wf6JtALZgvp2nM0lJesTA3tv46s7f+94offyU4w4uCJaR4UT7i9yktYoR4/AlFQ4sAxrZgT9m7+vGXUsEQWwF1TPwyGi8RY2xjrJri8m7gjz/A"
payload1["__VIEWSTATEGENERATOR"] = "38817EFB"
payload1["__VIEWSTATEENCRYPTED"] = ""
payload1["gvInmates$ob_gvInmatesSelectedRecordsContainer"] = "BOOKNO*_o_eg_*77882*_o_sep_*NAME*_o_eg_*GANNON%2C%20COURTNEY%20BETH*_o_sep_*BOOKDATE*_o_eg_*08%2F07%2F2019%2012%3A40*_o_sep_*ARSTAGENCY*_o_eg_*ACSO*_o_sep_*ARSTLOC*_o_eg_*SAN%20JOAQUIN%20CO*_o_sep_*STATUS*_o_eg_*In%20Custody*_o_osep_*true"
r2 = requests.post(url, headers=headers1, data=payload1)
print(r2.text)
Related
from twocaptcha import TwoCaptcha
import json
import requests
import random
def rand(list):
return random.randrange(0, len(list))
user_agents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246',
'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1'
]
headers = {
'user-agent': user_agents[rand(user_agents)],
"accept": "*/*",
"authority": "discord.com",
"method": "POST",
"path": "/api/v9/auth/register",
"scheme": "https",
"origin": "discord.com",
"referer": "discord.com/register",
"x-debug-options": "bugReporterEnabled",
"accept-language": "en-US,en;q=0.9",
"connection": "keep-alive",
"content-Type": "application/json",
"x-super-properties": "eyJvcyI6IldpbmRvd3MiLCJicm93c2VyIjoiRGlzY29yZCBDbGllbnQiLCJyZWxlYXNlX2NoYW5uZWwiOiJzdGFibGUiLCJjbGllbnRfdmVyc2lvbiI6IjEuMC45MDAzIiwib3NfdmVyc2lvbiI6IjEwLjAuMjIwMDAiLCJvc19hcmNoIjoieDY0Iiwic3lzdGVtX2xvY2FsZSI6ImVuLVVTIiwiY2xpZW50X2J1aWxkX251bWJlciI6MTA0OTY3LCJjbGllbnRfZXZlbnRfc291cmNlIjpudWxsfQ==",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin"
}
solver = TwoCaptcha('redacted')
try:
result = solver.hcaptcha(
sitekey='4c672d35-0701-42b2-88c3-78380b0db560',
url='https://discord.com/register',
)
except Exception as e:
sys.exit(e)
else:
print(result['code'])
def getfingerprint():
request_url = "https://discord.com/api/v9/experiments?with_guild_experiments=true"
r = requests.get(request_url , headers = {
"user-agent": user_agents[rand(user_agents)],
"x-context-properties": "eyJsb2NhdGlvbiI6Ii9jaGFubmVscy9AbWUifQ==",
"x-super-properties": "eyJvcyI6IldpbmRvd3MiLCJicm93c2VyIjoiQ2hyb21lIiwiZGV2aWNlIjoiIiwic3lzdGVtX2xvY2FsZSI6ImVuLVVTIiwiYnJvd3Nlcl91c2VyX2FnZW50IjoiTW96aWxsYS81LjAgKFdpbmRvd3MgTlQgMTAuMDsgV2luNjQ7IHg2NCkgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzEwNS4wLjAuMCBTYWZhcmkvNTM3LjM2IiwiYnJvd3Nlcl92ZXJzaW9uIjoiMTA1LjAuMC4wIiwib3NfdmVyc2lvbiI6IjEwIiwicmVmZXJyZXIiOiIiLCJyZWZlcnJpbmdfZG9tYWluIjoiIiwicmVmZXJyZXJfY3VycmVudCI6IiIsInJlZmVycmluZ19kb21haW5fY3VycmVudCI6IiIsInJlbGVhc2VfY2hhbm5lbCI6InN0YWJsZSIsImNsaWVudF9idWlsZF9udW1iZXIiOjE0NTQyOSwiY2xpZW50X2V2ZW50X3NvdXJjZSI6bnVsbH0="
})
if r.status_code == 200:
fingerprint = json.loads(r.text)["fingerprint"]
print(fingerprint)
else:
return r.text
payload = {
'captcha_key': result['code'],
'consent': True,
'date_of_birth': '2001-11-19',
'email': 'enter email',
'fingerprint': getfingerprint(),
'gift_code_sku_id': None,
'invite': None,
'password': 'enter username',
'username': 'enter password'
}
def register():
data = payload
res = requests.post('https://discord.com/api/v9/auth/register', headers = headers, json = data)
print('Token: ' + res.text)
if __name__ == '__main__':
register()
Code now works and generates accounts
just have to enter your own emails, username and password
can tell when it works cuz it outputs a token
if you get an error mentioning invalid hcaptcha response change email and use a vpn and try again
if you do decide to loop this have it loop slowly or setup proxies
I am learning scrapy and am trying to scrape this realtor site in Quebec. I am using their API to collect homes and print the URLs to the screen. But my last function print_urls() won't run. I really am stuck here i tried debugging it and it just skips right over my whole function block.
class CentrishomesSpider(scrapy.Spider):
name = 'centrisHomes'
# allowed_domains = ['www.centris.ca']
# start_urls = ['http://www.centris.ca/']
def start_requests(self):
query = {...
}
yield scrapy.Request(
url='https://www.centris.ca/property/UpdateQuery',
method='POST',
body=json.dumps(query),
headers={
'Content-Type': 'application/json',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
},
callback=self.get_inscriptions
)
...
def get_inscriptions(self, response):
resp, success = self.success(response)
if success == True:
print(Fore.GREEN + 'Query Updated' + Style.RESET_ALL)
else:
print(Fore.RED + 'Query Not Updated' + Style.RESET_ALL)
yield scrapy.Request(
url='https://www.centris.ca/Property/GetInscriptions',
method='POST',
body=json.dumps({"startPosition": 0}),
headers={
'Content-Type': 'application/json',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
},
callback=self.handle_inscriptions
)
def handle_inscriptions(self, response):
homes, success = self.success(response)
if success == True:
print(Fore.GREEN + 'Count ' + str(homes['d']['Result']['count']) + Style.RESET_ALL)
# self.test()
self.html = Selector(text=homes['d']['Result']['html'])
self.print_urls()
# print(response.body)
...
def success(self, response):
my_dict = literal_eval(response.body.decode(
'utf-8').replace(':true}', ':True}'))
if my_dict['d']['Succeeded'] == True:
return my_dict, True
else:
return False
def print_urls(self):
print('try')
# page_html = Selector(resp['d']['Result']['html'])
page_html = self.html
homes = page_html.xpath('//div[contains(#class, "property-thumbnail-item")]')
for home in homes:
yield{
'home_url':home.xpath('.//a[#class="property-thumbnail-summary-link"]/#href').get()
}
...
I figured out my own problem, it was because I turned my print_urls function into a generator, and calling self.print_urls() doesn't make my generator do anything. S/o #AbdealiJK I figured it out because of his answer.
https://stackoverflow.com/a/34609397/19966841
I try many method but they will not solve they show me the error:
All arrays must be of the same length
from bs4 import BeautifulSoup
import requests
import pandas as pd
review = []
ratings = []
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Accept-Encoding": "gzip, deflate",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"DNT": "1",
"Connection": "close",
"Upgrade-Insecure-Requests": "1",
}
for page in range(1, 5):
r = requests.get(
"https://www.amazon.com/s?k=redmi&page=2&qid=1631528810&ref=sr_pg_={page}".format(
page=page
),
headers=headers,
)
soup = BeautifulSoup(r.content, "lxml")
for d in soup.findAll("div", attrs={"class": "s-result-item"}):
rating = d.find("span", attrs={"class": "a-icon-alt"})
if rating is not None:
ratings.append(rating.text)
reviews = d.find("span", class_="a-size-base")
if reviews is not None:
review.append(reviews.text)
df = pd.DataFrame({"rating": ratings, "reviews": review})
df.to_csv("products .csv", index=False, encoding="utf-8")
ratings and reviews aren't the same length and you were scraping wrong containers. I made the necessary modifications, now it should work:
from bs4 import BeautifulSoup
import requests
import pandas as pd
review = []
ratings = []
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Accept-Encoding": "gzip, deflate",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"DNT": "1",
"Connection": "close",
"Upgrade-Insecure-Requests": "1",
}
for page in range(1, 5):
cookies = {'session': '17ab96bd8ffbe8ca58a78657a918558'}
r = requests.get(
"https://www.amazon.com/s?k=redmi&page=2&qid=1631528810&ref=sr_pg_={page}".format(
page=page
),
headers=headers,
cookies =cookies
)
soup = BeautifulSoup(r.content, "lxml")
for d in soup.select(".s-result-item[data-component-type='s-search-result']"):
rating = d.find("span", attrs={"class": "a-icon-alt"})
if rating is not None:
ratings.append(rating.text)
else:
ratings.append("-")
reviews = d.find("span", class_="a-size-base")
if reviews is not None and rating is not None:
review.append(reviews.text)
else:
review.append("-")
df = pd.DataFrame({"rating": ratings, "reviews": review})
df.to_csv("products.csv", index=False, encoding="utf-8")
i'm sending below request to URL and get the response from it
import requests
url = "http://localhost/dat.txt"
payload = {}
headers = {
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
'Sec-Fetch-Dest': 'document',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
}
response = requests.request("GET", url, headers=headers, data = payload)
print(response.text.encode('utf8'))
Below is the response data that I get -
mohame4|nameon#example.com|passsd!##$4|head,customer|manager,devlop
mohame3|nameon3#example.com|passsd!##$4|head,customer|manager,devlop
I do this with the data
for i in response.text:
try:
i = i.strip().split('|')
userna = i[0]
emaill = i[1]
passd = i[2]
rol1= i[3]
rol2= i[4]
except:
pass
How can I make rol1 as
this head,customer
to
rol1=['head','customer']
Simply split the string you're getting:
rol1 = i[3].split(',')
You could do this more... gracefully, though, using iterable unpacking:
username, email, password, rol1, rol2 = i.strip().split('|')
rol1 = rol1.split(',')
thanks for all helper special #ForceBru
import requests
url = "http://localhost/dat.txt"
response = requests.request("GET", url)
print(response.text)
dat = str(response.text).split('\n')
for i in dat:
i = i.strip().split('|')
print(i[3].split(","))
# TODO: write code...
I'm trying to scrape WizzAir for personal use. Can't understand what's wrong with my code. Could it be incorrect payload object or cookies?
import requests
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36",
"Accept": "application/json, text/plain, */*",
"Accept-Encoding": "gzip, deflate, sdch, br",
"Accept-Language": "en-US,en;q=0.8,lt;q=0.6,ru;q=0.4",
"Origin": "https://wizzair.com",
"Referer": "https://wizzair.com/"
}
search_url = "https://wizzair.com/lt-LT/FlightSearch"
session = requests.Session()
r = session.get("https://be.wizzair.com/3.8.2/Api/asset/yellowRibbon", headers=headers, allow_redirects=False)
session_id = r.cookies["ASP.NET_SessionId"]
cookies = {
"ASP.NET_SessionId": session_id,
"HomePageSelector": "FlightSearch",
}
# wizz_url = "https://be.wizzair.com/3.8.2/Api/search/search"
wizz_url = "https://be.wizzair.com/3.8.2/Api/asset/farechart"
payload = {"flightList":[{"departureStation":"VNO","arrivalStation":"FCO","departureDate":"2017-02-20"}],"adultCount":1,"childCount":0,"infantCount":0,"wdc":True, "dayInterval":3}
r = session.post(url=wizz_url,data=payload,headers=headers, cookies=cookies)
print r.content
>>> {"validationCodes":["FlightCount_MustBe_OneOrTwo"]}
I run this - even without session and cookies - and get some data.
You have to send it as JSON - using json=payload
import requests
payload = {
"flightList":[
{
"departureStation": "VNO",
"arrivalStation": "FCO",
"departureDate": "2017-02-20"
}
],
"adultCount": 1,
"childCount": 0,
"infantCount": 0,
"wdc": True,
"dayInterval": 3
}
url = 'https://be.wizzair.com/3.8.2/Api/search/search'
r = requests.post(url, json=payload)
print(r.text)
data = r.json()
print(data['outboundFlights'][0]['flightNumber'])
If you will have to use cookies and headers then use Session and then you don't have to copy cookies and headers from one request to another.
import requests
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36",
#"Accept": "application/json, text/plain, */*",
#"Accept-Encoding": "gzip, deflate, sdch, br",
#"Accept-Language": "en-US,en;q=0.8,lt;q=0.6,ru;q=0.4",
}
s = requests.Session()
s.headers.update(headers)
# to get cookies
r = s.get("https://www.wizzair.com/")
payload = {
"flightList":[
{
"departureStation": "VNO",
"arrivalStation": "FCO",
"departureDate": "2017-02-20"
}
],
"adultCount": 1,
"childCount": 0,
"infantCount": 0,
"wdc": True,
"dayInterval": 3
}
url = 'https://be.wizzair.com/3.8.2/Api/search/search'
r = s.post(url, json=payload)
print(r.text)
data = r.json()
print(data['outboundFlights'][0]['flightNumber'])