Month: December 2016

google map api

https://developers.google.com/maps/documentation/geocoding/get-api-key

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import httplib2
import json
 
def getGeocodeLocation(inputString):
    google_api_key = "your_key"
    locationString = inputString.replace(" ", "+")
    url = ('https://maps.googleapis.com/maps/api/geocode/json?address=%s&key=%s'% (
        locationString, google_api_key))
    h = httplib2.Http()
    response, content = h.request(url, 'GET')
    result = json.loads(content)
 
    latitude = result['result'][0]['geometry']['location']['lat']
    longitude = result['result'][0]['geometry']['location']['lng']
    return(latitude,longitude)

flask

1
2
3
4
5
6
7
8
9
10
11
12
from flask import Flask
app = Flask(__name__)
 
def puppyFunction():
    return "Yes, puppies!"
 
def puppiesFunction(id)
    return "This method will act on the puppy with id %s" % id
 
if __name__ == '__main__':
    app.debug = True
    app.run(host='0.0.0.0', port=5000)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy import create_engine
from passlib.apps import custom_app_context as pwd_context
 
Base = declarative_base()
 
class User(Base):
    __tablename__ = 'user'
    id = Column(Integer, primary_key=True)
    username = Column(String(32), index=True)
    password_hash = Column(String(64))
 
engine = create_engine('sqlite:///users.db')

port:5000 python server

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#THIS IS A WEBSERVER FOR DEMONSTRATING THE TYPES OF RESPONSES WE SEE FROM AN API ENDPOINT
from flask import Flask
app = Flask(__name__)
 
#GET REQUEST
 
@app.route('/readHello')
def getRequestHello():
    return "Hi, I got your GET Request!"
 
#POST REQUEST
@app.route('/createHello', methods = ['POST'])
def postRequestHello():
    return "I see you sent a POST message :-)"
#UPDATE REQUEST
@app.route('/updateHello', methods = ['PUT'])
def updateRequestHello():
    return "Sending Hello on an PUT request!"
 
#DELETE REQUEST
@app.route('/deleteHello', methods = ['DELETE'])
def deleteRequestHello():
    return "Deleting your hard drive.....haha just kidding! I received a DELETE request!"
 
if __name__ == '__main__':
    app.debug = True
    app.run(host='0.0.0.0', port=5000)

Caching

db-read() -> 100ms

if request in cache,
return cache[request]

cashe is a hashtable
basic cache algorithm is below

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
import time
 
def complex_computation(a, b):
    time.sleep(.5)
    return a + b
 
cache = {}
def cached_computation(a, b):
    key = (a, b)
    if key in cache:
        r = cache[key]
    else:
        r = complex_computation(a, b)
        cache[key] = r
    return r
 
start_time = time.time()
print cached_computation(5, 3)
print "the first computation took %f second" % (time.time() - start_time)

when db query can be cashed in serverside as dictionary like below.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
CACHE = {}
def top_arts();
        key = 'top'
        if key in CACHE:
            arts = CHACHE[key]
        else:
            loggin.error("DB QUERY")
            art = db.GqlQuery("SELECT * From Art "
                            "WHERE ancestor is :1"
                            "ORDER BY DESC")
 
        arts = list(arts)
        CACHE[key] = arts
    return arts

when posted, cache cleared.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
CACHE = {}
def top_arts(update = False);
        key = 'top'
        if not update and key in CACHE:
            arts = CHACHE[key]
        else:
            loggin.error("DB QUERY")
            art = db.GqlQuery("SELECT * From Art "
                            "WHERE ancestor is :1"
                            "ORDER BY DESC")
 
        arts = list(arts)
        CACHE[key] = arts
    return arts

DB read should be only submission.
loadbalancer also can handle large traffic.

load balancere

1
2
3
4
5
n = -1
def get_server():
    global n
    n += 1
    return SERVERS[n % len(SERVERS)]

handler, url mapping DBmodel

parsing

>>> import urllib2
>>> import urllib
>>> p = urllib2.urlopen("http://www.google.com")
>>> p
>
>>> c = p.read()
>>> dir(p)
['__doc__', '__init__', '__iter__', '__module__', '__repr__', 'close', 'code', 'fileno', 'fp', 'getcode', 'geturl', 'headers', 'info', 'msg', 'next', 'read', 'readline', 'readlines', 'url']
>>> p.url
'http://www.google.co.jp/?gfe_rd=cr&ei=MtZkWNDXDYSL8QeD46cY'
>>> p.headers

>>> p.headers.items()
[('x-xss-protection', '1; mode=block'), ('set-cookie', 'NID=93=O65u9flBWzM92U9MzcezfIXaeG9itO-ala3ogt6T7fipovY5ily4QBNUxbUbsVga_hYeJEKWDq891mFaPgZm2Ya_1gvUZm37K2pNfFpOUVxCptVtOSAn3OXvUHCzKBaC; expires=Fri, 30-Jun-2017 09:24:02 GMT; path=/; domain=.google.co.jp; HttpOnly'), ('accept-ranges', 'none'), ('expires', '-1'), ('vary', 'Accept-Encoding'), ('server', 'gws'), ('connection', 'close'), ('cache-control', 'private, max-age=0'), ('date', 'Thu, 29 Dec 2016 09:24:02 GMT'), ('p3p', 'CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."'), ('content-type', 'text/html; charset=Shift_JIS'), ('x-frame-options', 'SAMEORIGIN')]
>>> p.headers['content-type']
'text/html; charset=Shift_JIS'

>>> s = = urllib2.urlopen("http://www.example.com")
SyntaxError: invalid syntax
>>> s = urllib2.urlopen("http://www.example.com")
>>> s.url
'http://www.example.com'
>>> s.headers.items()
[('content-length', '1270'), ('x-ec-custom-error', '1'), ('x-cache', 'HIT'), ('expires', 'Thu, 05 Jan 2017 09:28:12 GMT'), ('vary', 'Accept-Encoding'), ('server', 'ECS (rhv/818F)'), ('last-modified', 'Fri, 09 Aug 2013 23:54:35 GMT'), ('connection', 'close'), ('etag', '"359670651+gzip+ident"'), ('cache-control', 'max-age=604800'), ('date', 'Thu, 29 Dec 2016 09:28:12 GMT'), ('content-type', 'text/html')]

parsing xml
from xml.com import minidom

json

>>> import json
>>> j = '{"one": 1, "numbers": [1,2,3.5]}'
>>> json.loads(j)
{u'numbers': [1, 2, 3.5], u'one': 1}
1
2
3
def total_ups():
    j = json.loads(reddit_front)
    sum(c['data']['ups'] for c in j['data']['children'])

host ip info
host ip info

1
2
3
4
5
6
7
8
9
10
11
12
13
IP_URL = "http://api.hostip.info/?ip="
def get_coords(ip):
    url = IP_URL + ip
    content = None
    content = urllib2.urlopen(url).read()
    except URLError:
        return
    if content:
        d = minidom.parseString(content)
        coords = d.getElementByTagName("gml:coordinates")
        if coords and coords[0].childNodes[0].nodeValue:
            lon, lat = coords[0].childNodes[0].nodeValue.split(',')
            return db.GetPt(lat, lon)

Gmap

1
2
3
4
5
6
7
8
GMAPS_URL = ""
 
def gmap_img(points):
    markers = '&'.join('makers=%s,%s' % (p.lat, p.lon)
        for p in points)
    return GMAPS_URL + markers
 
print gmaps_img([Point])

Hashing

What is a hash?
H(x) -> y

ex. crc32 – checksums
md5 – fast
sha1 – secure
sha256 -pretty good

set-cookie:visit = 5, [hash]

making a hash

1
2
3
4
5
6
7
import hashlib
 
def hash_str(s):
    return hashlib.md5(s).hexdigest()
 
def make_secure_val(s):
    return "%s, %s" % (s, hash_string(s))

checking correct hash

1
2
3
4
def check_secure_val(h):
    val = h.split('.')[0]
    if h == make_secure_val(val):
        return val
1
2
3
4
5
6
7
8
9
10
def get(self):
    self.response.headers['Content-Type'] = 'text/plain'
    visits = 0
    visits = self.request.cookies.get('visits')
    if visit_cookie_val:
        cookie_val = check_secure_val(visit_cookie_str)
        if cookie_val:
            visits = ini(cookie_val)
     
    visits += 1

HMAC is hash-based message authentication code
hmac(secret, key, h)-> [HASH]

$ hmac.new(“secret”, “hoge”).hexdigest()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
import hashlib
import hmac
 
SECRET = 'imsosecret'
def hash_str(s):
    return hmac.new(SECRET, s).hexdigest()
 
def make_secure_val(s):
    return "%s|%s" % (s, hash_str(s))
 
def check_secure_val(h):
    val = h.split('|')[0]
    if h == make_secure_val(val):
        return val

database should change password hashing
random function in python

1
2
def make_salt():
    return ''.(random.choice(string.letters) for x in xrange(5))
1
2
3
4
def make_pw_hash(name, pw):
    salt = make_salt()
    h = hashlib.sha256(name + pw * salt).hexdigest()
    return '%s,%s' % (h, salt)

Python Set-Cookie

Set-Cookie: name=steve; Domain=www.rddit.com; Path=/

domain is restricted wwww.

Third party set cookie such as google analytics
ad network also set cookie

Set-Cookie: user=123; Expire= Ture, 1 Jan
"session" cookie = no Expire

session cookie delete when close the browser.

1
2
3
4
5
6
7
8
9
10
11
12
def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        visits = self.request.cookies.get('visits', 0)
        if visits.isdigit();
            visits = int(visits) + 1
        else:
            visits = 0
        self.response.headers.add_header('Set-Cookie', 'visits=%s' % visits)
        if visits > 100:
            self.write("you are the best ever")
        else:
            self.write("you've been here %s times!" % visits)

write document.cookie in console.
we can rewrite cookie.

document.cookie

"wordpress_test_cookie=WP+Cookie+check; wp-settings-1=editor%3Dhtml%26libraryContent%3Dbrowse%26imgsize%3Dmedium; wp-settings-time-1=1482994300; _ga=GA1.2.511761152.1479929467"

document.cookie="wp-settings-tims-1=1482994301"

"wp-settings-tims-1=1482994301"

db connection

1
2
3
4
5
6
7
8
9
db. = sqlite3.connect(':memory:')
db.execute('create table links ' +
            '(id integer, submitter_id integer, submitted_time integer, ' +
            'votes integer, title text, url text)')
for l in links:
    db.execute('insert into links values (?, ?, ?, ?, ?, ?)', l)
 
def query():
    c = db.execute("select * from links")
1
2
3
4
5
6
def query():
    cursor = db.execute("select * from links")
    for link_tuple in cursor:
        link = Link(*link_tuple)
 
print query()
1
2
3
4
def query():
    cursor = db.execute("select * from links where submitter_id = 62443 and votes > 1000")
    link = Link(*c.fetchone())
    return link.id

order by

1
2
3
4
5
6
def query():
    cursor = db.execute("select * from links where submitter_id = 62443 order by submitter_time asc")
    for link_tuple in c:
        link = Link(*link_tuple)
        results.append(link.id)
    return results

join

1
2
3
4
def link_by_id(link_id):
    for l in links:
        if l.id == link_id:
            return l

scaling database is to 1.replicate, 2.shared.