Using Beautiful Soup

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
import requests
from bs4 import BeautifulSoup
import json
 
html_page = "page_source.html"
 
def extract_data(page):
    data = {"eventvalidation":"",
            "viewstate": ""}
    with open(page, "r") as html;
        soup = BeautifulSoup(html, "lxml")
        ev = soup.find(id="__EVENTVALIDATION")
        data["eventvalidation"] = ev["value"]
 
        vs = soup.find(id="__VIEWSTATE")
        data["viewstate"] = vs["value"]
 
        return data
 
def make_request(data):
    eventvalidation = data["eventvalidation"]
    viewstate = data["viewstate"]
 
r = requests.post("http://www.transtats.bts.gov/Data_Elements.aspx?Data=2",
            data={'AirportList' : "BOS",
                    'CarrierList' : "VX",
                    'Submit' : "Submit",
                    '__EVENTTARGET' : "",
                    '__EVENTARGUMENT' : "",
                    '__EVENTVALIDATION' : eventvalidation,
                    '__VIEWSTATE' : viewstate})
 
    return r.text
 
def test():